PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42 my $OVMF_IMG = '/usr/share/kvm/OVMF-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 # Note about locking: we use flock on the config file protect
  49 # against concurent actions.
  50 # Aditionaly, we have a 'lock' setting in the config file. This
  51 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  52 # allowed when such lock is set. But you can ignore this kind of
  53 # lock with the --skiplock flag.
  54
  55 cfs_register_file('/qemu-server/',
  56                   \&parse_vm_config,
  57                   \&write_vm_config);
  58
  59 PVE::JSONSchema::register_standard_option('skiplock', {
  60     description => "Ignore locks - only root is allowed to use this option.",
  61     type => 'boolean',
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  66     description => "Some command save/restore state from this location.",
  67     type => 'string',
  68     maxLength => 128,
  69     optional => 1,
  70 });
  71
  72 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  73     description => "The name of the snapshot.",
  74     type => 'string', format => 'pve-configid',
  75     maxLength => 40,
  76 });
  77
  78 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  79     type => 'string',
  80     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  81     description => "The drive's backing file's data format.",
  82     optional => 1,
  83 });
  84
  85 #no warnings 'redefine';
  86
  87 sub cgroups_write {
  88    my ($controller, $vmid, $option, $value) = @_;
  89
  90    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  91    PVE::ProcFSTools::write_proc_entry($path, $value);
  92
  93 }
  94
  95 my $nodename = PVE::INotify::nodename();
  96
  97 mkdir "/etc/pve/nodes/$nodename";
  98 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  99 mkdir $confdir;
 100
 101 my $var_run_tmpdir = "/var/run/qemu-server";
 102 mkdir $var_run_tmpdir;
 103
 104 my $lock_dir = "/var/lock/qemu-server";
 105 mkdir $lock_dir;
 106
 107 my $pcisysfs = "/sys/bus/pci";
 108
 109 my $cpu_vendor_list = {
 110     # Intel CPUs
 111     486 => 'GenuineIntel',
 112     pentium => 'GenuineIntel',
 113     pentium2  => 'GenuineIntel',
 114     pentium3  => 'GenuineIntel',
 115     coreduo => 'GenuineIntel',
 116     core2duo => 'GenuineIntel',
 117     Conroe  => 'GenuineIntel',
 118     Penryn  => 'GenuineIntel',
 119     Nehalem  => 'GenuineIntel',
 120     Westmere => 'GenuineIntel',
 121     SandyBridge => 'GenuineIntel',
 122     IvyBridge => 'GenuineIntel',
 123     Haswell => 'GenuineIntel',
 124     'Haswell-noTSX' => 'GenuineIntel',
 125     Broadwell => 'GenuineIntel',
 126     'Broadwell-noTSX' => 'GenuineIntel',
 127
 128     # AMD CPUs
 129     athlon => 'AuthenticAMD',
 130     phenom  => 'AuthenticAMD',
 131     Opteron_G1  => 'AuthenticAMD',
 132     Opteron_G2  => 'AuthenticAMD',
 133     Opteron_G3  => 'AuthenticAMD',
 134     Opteron_G4  => 'AuthenticAMD',
 135     Opteron_G5  => 'AuthenticAMD',
 136
 137     # generic types, use vendor from host node
 138     host => 'default',
 139     kvm32 => 'default',
 140     kvm64 => 'default',
 141     qemu32 => 'default',
 142     qemu64 => 'default',
 143 };
 144
 145 my $cpu_fmt = {
 146     cputype => {
 147         description => "Emulated CPU type.",
 148         type => 'string',
 149         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 150         default => 'kvm64',
 151         default_key => 1,
 152     },
 153     hidden => {
 154         description => "Do not identify as a KVM virtual machine.",
 155         type => 'boolean',
 156         optional => 1,
 157         default => 0
 158     },
 159 };
 160
 161 my $watchdog_fmt = {
 162     model => {
 163         default_key => 1,
 164         type => 'string',
 165         enum => [qw(i6300esb ib700)],
 166         description => "Watchdog type to emulate.",
 167         default => 'i6300esb',
 168         optional => 1,
 169     },
 170     action => {
 171         type => 'string',
 172         enum => [qw(reset shutdown poweroff pause debug none)],
 173         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 174         optional => 1,
 175     },
 176 };
 177 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 178
 179 my $confdesc = {
 180     onboot => {
 181         optional => 1,
 182         type => 'boolean',
 183         description => "Specifies whether a VM will be started during system bootup.",
 184         default => 0,
 185     },
 186     autostart => {
 187         optional => 1,
 188         type => 'boolean',
 189         description => "Automatic restart after crash (currently ignored).",
 190         default => 0,
 191     },
 192     hotplug => {
 193         optional => 1,
 194         type => 'string', format => 'pve-hotplug-features',
 195         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 196         default => 'network,disk,usb',
 197     },
 198     reboot => {
 199         optional => 1,
 200         type => 'boolean',
 201         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 202         default => 1,
 203     },
 204     lock => {
 205         optional => 1,
 206         type => 'string',
 207         description => "Lock/unlock the VM.",
 208         enum => [qw(migrate backup snapshot rollback)],
 209     },
 210     cpulimit => {
 211         optional => 1,
 212         type => 'number',
 213         description => "Limit of CPU usage.",
 214         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 215         minimum => 0,
 216         maximum => 128,
 217         default => 0,
 218     },
 219     cpuunits => {
 220         optional => 1,
 221         type => 'integer',
 222         description => "CPU weight for a VM.",
 223         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 224         minimum => 0,
 225         maximum => 500000,
 226         default => 1024,
 227     },
 228     memory => {
 229         optional => 1,
 230         type => 'integer',
 231         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 232         minimum => 16,
 233         default => 512,
 234     },
 235     balloon => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 239         minimum => 0,
 240     },
 241     shares => {
 242         optional => 1,
 243         type => 'integer',
 244         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 245         minimum => 0,
 246         maximum => 50000,
 247         default => 1000,
 248     },
 249     keyboard => {
 250         optional => 1,
 251         type => 'string',
 252         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 253         enum => PVE::Tools::kvmkeymaplist(),
 254         default => 'en-us',
 255     },
 256     name => {
 257         optional => 1,
 258         type => 'string', format => 'dns-name',
 259         description => "Set a name for the VM. Only used on the configuration web interface.",
 260     },
 261     scsihw => {
 262         optional => 1,
 263         type => 'string',
 264         description => "SCSI controller model",
 265         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 266         default => 'lsi',
 267     },
 268     description => {
 269         optional => 1,
 270         type => 'string',
 271         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 272     },
 273     ostype => {
 274         optional => 1,
 275         type => 'string',
 276         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 277         description => "Specify guest operating system.",
 278         verbose_description => <<EODESC,
 279 Specify guest operating system. This is used to enable special
 280 optimization/features for specific operating systems:
 281
 282 [horizontal]
 283 other;; unspecified OS
 284 wxp;; Microsoft Windows XP
 285 w2k;; Microsoft Windows 2000
 286 w2k3;; Microsoft Windows 2003
 287 w2k8;; Microsoft Windows 2008
 288 wvista;; Microsoft Windows Vista
 289 win7;; Microsoft Windows 7
 290 win8;; Microsoft Windows 8/2012
 291 l24;; Linux 2.4 Kernel
 292 l26;; Linux 2.6/3.X Kernel
 293 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 294 EODESC
 295     },
 296     boot => {
 297         optional => 1,
 298         type => 'string',
 299         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 300         pattern => '[acdn]{1,4}',
 301         default => 'cdn',
 302     },
 303     bootdisk => {
 304         optional => 1,
 305         type => 'string', format => 'pve-qm-bootdisk',
 306         description => "Enable booting from specified disk.",
 307         pattern => '(ide|sata|scsi|virtio)\d+',
 308     },
 309     smp => {
 310         optional => 1,
 311         type => 'integer',
 312         description => "The number of CPUs. Please use option -sockets instead.",
 313         minimum => 1,
 314         default => 1,
 315     },
 316     sockets => {
 317         optional => 1,
 318         type => 'integer',
 319         description => "The number of CPU sockets.",
 320         minimum => 1,
 321         default => 1,
 322     },
 323     cores => {
 324         optional => 1,
 325         type => 'integer',
 326         description => "The number of cores per socket.",
 327         minimum => 1,
 328         default => 1,
 329     },
 330     numa => {
 331         optional => 1,
 332         type => 'boolean',
 333         description => "Enable/disable NUMA.",
 334         default => 0,
 335     },
 336     hugepages => {
 337         optional => 1,
 338         type => 'string',
 339         description => "Enable/disable hugepages memory.",
 340         enum => [qw(any 2 1024)],
 341     },
 342     vcpus => {
 343         optional => 1,
 344         type => 'integer',
 345         description => "Number of hotplugged vcpus.",
 346         minimum => 1,
 347         default => 0,
 348     },
 349     acpi => {
 350         optional => 1,
 351         type => 'boolean',
 352         description => "Enable/disable ACPI.",
 353         default => 1,
 354     },
 355     agent => {
 356         optional => 1,
 357         type => 'boolean',
 358         description => "Enable/disable Qemu GuestAgent.",
 359         default => 0,
 360     },
 361     kvm => {
 362         optional => 1,
 363         type => 'boolean',
 364         description => "Enable/disable KVM hardware virtualization.",
 365         default => 1,
 366     },
 367     tdf => {
 368         optional => 1,
 369         type => 'boolean',
 370         description => "Enable/disable time drift fix.",
 371         default => 0,
 372     },
 373     localtime => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 377     },
 378     freeze => {
 379         optional => 1,
 380         type => 'boolean',
 381         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 382     },
 383     vga => {
 384         optional => 1,
 385         type => 'string',
 386         description => "Select the VGA type.",
 387         verbose_description => "Select the VGA type. If you want to use high resolution" .
 388             " modes (>= 1280x1024x16) then you should use the options " .
 389             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 390             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 391             "display sever. For win* OS you can select how many independent " .
 392             "displays you want, Linux guests can add displays them self. " .
 393             "You can also run without any graphic card, using a serial device" .
 394             " as terminal.",
 395         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 396     },
 397     watchdog => {
 398         optional => 1,
 399         type => 'string', format => 'pve-qm-watchdog',
 400         description => "Create a virtual hardware watchdog device.",
 401         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 402             " (by a guest action), the watchdog must be periodically polled " .
 403             "by an agent inside the guest or else the watchdog will reset " .
 404             "the guest (or execute the respective action specified)",
 405     },
 406     startdate => {
 407         optional => 1,
 408         type => 'string',
 409         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 410         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 411         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 412         default => 'now',
 413     },
 414     startup =>  get_standard_option('pve-startup-order'),
 415     template => {
 416         optional => 1,
 417         type => 'boolean',
 418         description => "Enable/disable Template.",
 419         default => 0,
 420     },
 421     args => {
 422         optional => 1,
 423         type => 'string',
 424         description => "Arbitrary arguments passed to kvm.",
 425         verbose_description => <<EODESCR,
 426 Arbitrary arguments passed to kvm, for example:
 427
 428 args: -no-reboot -no-hpet
 429
 430 NOTE: this option is for experts only.
 431 EODESCR
 432     },
 433     tablet => {
 434         optional => 1,
 435         type => 'boolean',
 436         default => 1,
 437         description => "Enable/disable the USB tablet device.",
 438         verbose_description => "Enable/disable the USB tablet device. This device is " .
 439             "usually needed to allow absolute mouse positioning with VNC. " .
 440             "Else the mouse runs out of sync with normal VNC clients. " .
 441             "If you're running lots of console-only guests on one host, " .
 442             "you may consider disabling this to save some context switches. " .
 443             "This is turned off by default if you use spice (-vga=qxl).",
 444     },
 445     migrate_speed => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 449         minimum => 0,
 450         default => 0,
 451     },
 452     migrate_downtime => {
 453         optional => 1,
 454         type => 'number',
 455         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 456         minimum => 0,
 457         default => 0.1,
 458     },
 459     cdrom => {
 460         optional => 1,
 461         type => 'string', format => 'pve-qm-ide',
 462         typetext => '<volume>',
 463         description => "This is an alias for option -ide2",
 464     },
 465     cpu => {
 466         optional => 1,
 467         description => "Emulated CPU type.",
 468         type => 'string',
 469         format => $cpu_fmt,
 470     },
 471     parent => get_standard_option('pve-snapshot-name', {
 472         optional => 1,
 473         description => "Parent snapshot name. This is used internally, and should not be modified.",
 474     }),
 475     snaptime => {
 476         optional => 1,
 477         description => "Timestamp for snapshots.",
 478         type => 'integer',
 479         minimum => 0,
 480     },
 481     vmstate => {
 482         optional => 1,
 483         type => 'string', format => 'pve-volume-id',
 484         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 485     },
 486     machine => {
 487         description => "Specific the Qemu machine type.",
 488         type => 'string',
 489         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 490         maxLength => 40,
 491         optional => 1,
 492     },
 493     smbios1 => {
 494         description => "Specify SMBIOS type 1 fields.",
 495         type => 'string', format => 'pve-qm-smbios1',
 496         maxLength => 256,
 497         optional => 1,
 498     },
 499     protection => {
 500         optional => 1,
 501         type => 'boolean',
 502         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 503         default => 0,
 504     },
 505     bios => {
 506         optional => 1,
 507         type => 'string',
 508         enum => [ qw(seabios ovmf) ],
 509         description => "Select BIOS implementation.",
 510         default => 'seabios',
 511     },
 512 };
 513
 514 # what about other qemu settings ?
 515 #cpu => 'string',
 516 #machine => 'string',
 517 #fda => 'file',
 518 #fdb => 'file',
 519 #mtdblock => 'file',
 520 #sd => 'file',
 521 #pflash => 'file',
 522 #snapshot => 'bool',
 523 #bootp => 'file',
 524 ##tftp => 'dir',
 525 ##smb => 'dir',
 526 #kernel => 'file',
 527 #append => 'string',
 528 #initrd => 'file',
 529 ##soundhw => 'string',
 530
 531 while (my ($k, $v) = each %$confdesc) {
 532     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 533 }
 534
 535 my $MAX_IDE_DISKS = 4;
 536 my $MAX_SCSI_DISKS = 14;
 537 my $MAX_VIRTIO_DISKS = 16;
 538 my $MAX_SATA_DISKS = 6;
 539 my $MAX_USB_DEVICES = 5;
 540 my $MAX_NETS = 32;
 541 my $MAX_UNUSED_DISKS = 8;
 542 my $MAX_HOSTPCI_DEVICES = 4;
 543 my $MAX_SERIAL_PORTS = 4;
 544 my $MAX_PARALLEL_PORTS = 3;
 545 my $MAX_NUMA = 8;
 546
 547 my $numa_fmt = {
 548     cpus => {
 549         type => "string",
 550         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 551         description => "CPUs accessing this NUMA node.",
 552         format_description => "id[-id];...",
 553     },
 554     memory => {
 555         type => "number",
 556         description => "Amount of memory this NUMA node provides.",
 557         optional => 1,
 558     },
 559     hostnodes => {
 560         type => "string",
 561         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 562         description => "Host NUMA nodes to use.",
 563         format_description => "id[-id];...",
 564         optional => 1,
 565     },
 566     policy => {
 567         type => 'string',
 568         enum => [qw(preferred bind interleave)],
 569         description => "NUMA allocation policy.",
 570         optional => 1,
 571     },
 572 };
 573 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 574 my $numadesc = {
 575     optional => 1,
 576     type => 'string', format => $numa_fmt,
 577     description => "NUMA topology.",
 578 };
 579 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 580
 581 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 582     $confdesc->{"numa$i"} = $numadesc;
 583 }
 584
 585 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 586                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 587                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 588 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 589
 590 my $net_fmt_bridge_descr = <<__EOD__;
 591 Bridge to attach the network device to. The Proxmox VE standard bridge
 592 is called 'vmbr0'.
 593
 594 If you do not specify a bridge, we create a kvm user (NATed) network
 595 device, which provides DHCP and DNS services. The following addresses
 596 are used:
 597
 598  10.0.2.2   Gateway
 599  10.0.2.3   DNS Server
 600  10.0.2.4   SMB Server
 601
 602 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 603 __EOD__
 604
 605 my $net_fmt = {
 606     macaddr => {
 607         type => 'string',
 608         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 609         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 610         format_description => "XX:XX:XX:XX:XX:XX",
 611         optional => 1,
 612     },
 613     model => {
 614         type => 'string',
 615         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 616         enum => $nic_model_list,
 617         default_key => 1,
 618     },
 619     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 620     bridge => {
 621         type => 'string',
 622         description => $net_fmt_bridge_descr,
 623         format_description => 'bridge',
 624         optional => 1,
 625     },
 626     queues => {
 627         type => 'integer',
 628         minimum => 0, maximum => 16,
 629         description => 'Number of packet queues to be used on the device.',
 630         optional => 1,
 631     },
 632     rate => {
 633         type => 'number',
 634         minimum => 0,
 635         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 636         optional => 1,
 637     },
 638     tag => {
 639         type => 'integer',
 640         minimum => 1, maximum => 4094,
 641         description => 'VLAN tag to apply to packets on this interface.',
 642         optional => 1,
 643     },
 644     trunks => {
 645         type => 'string',
 646         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 647         description => 'VLAN trunks to pass through this interface.',
 648         format_description => 'vlanid[;vlanid...]',
 649         optional => 1,
 650     },
 651     firewall => {
 652         type => 'boolean',
 653         description => 'Whether this interface should be protected by the firewall.',
 654         optional => 1,
 655     },
 656     link_down => {
 657         type => 'boolean',
 658         description => 'Whether this interface should be disconnected (like pulling the plug).',
 659         optional => 1,
 660     },
 661 };
 662
 663 my $netdesc = {
 664     optional => 1,
 665     type => 'string', format => $net_fmt,
 666     description => "Specify network devices.",
 667 };
 668
 669 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 670
 671 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 672     $confdesc->{"net$i"} = $netdesc;
 673 }
 674
 675 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 676 sub verify_volume_id_or_qm_path {
 677     my ($volid, $noerr) = @_;
 678
 679     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 680         return $volid;
 681     }
 682
 683     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 684     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 685     if ($@) {
 686         return undef if $noerr;
 687         die $@;
 688     }
 689     return $volid;
 690 }
 691
 692 my $drivename_hash;
 693
 694 my %drivedesc_base = (
 695     volume => { alias => 'file' },
 696     file => {
 697         type => 'string',
 698         format => 'pve-volume-id-or-qm-path',
 699         default_key => 1,
 700         format_description => 'volume',
 701         description => "The drive's backing volume.",
 702     },
 703     media => {
 704         type => 'string',
 705         enum => [qw(cdrom disk)],
 706         description => "The drive's media type.",
 707         default => 'disk',
 708         optional => 1
 709     },
 710     cyls => {
 711         type => 'integer',
 712         description => "Force the drive's physical geometry to have a specific cylinder count.",
 713         optional => 1
 714     },
 715     heads => {
 716         type => 'integer',
 717         description => "Force the drive's physical geometry to have a specific head count.",
 718         optional => 1
 719     },
 720     secs => {
 721         type => 'integer',
 722         description => "Force the drive's physical geometry to have a specific sector count.",
 723         optional => 1
 724     },
 725     trans => {
 726         type => 'string',
 727         enum => [qw(none lba auto)],
 728         description => "Force disk geometry bios translation mode.",
 729         optional => 1,
 730     },
 731     snapshot => {
 732         type => 'boolean',
 733         description => "Whether the drive should be included when making snapshots.",
 734         optional => 1,
 735     },
 736     cache => {
 737         type => 'string',
 738         enum => [qw(none writethrough writeback unsafe directsync)],
 739         description => "The drive's cache mode",
 740         optional => 1,
 741     },
 742     format => get_standard_option('pve-qm-image-format'),
 743     size => {
 744         type => 'string',
 745         format => 'disk-size',
 746         format_description => 'DiskSize',
 747         description => "Disk size. This is purely informational and has no effect.",
 748         optional => 1,
 749     },
 750     backup => {
 751         type => 'boolean',
 752         description => "Whether the drive should be included when making backups.",
 753         optional => 1,
 754     },
 755     werror => {
 756         type => 'string',
 757         enum => [qw(enospc ignore report stop)],
 758         description => 'Write error action.',
 759         optional => 1,
 760     },
 761     aio => {
 762         type => 'string',
 763         enum => [qw(native threads)],
 764         description => 'AIO type to use.',
 765         optional => 1,
 766     },
 767     discard => {
 768         type => 'string',
 769         enum => [qw(ignore on)],
 770         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 771         optional => 1,
 772     },
 773     detect_zeroes => {
 774         type => 'boolean',
 775         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 776         optional => 1,
 777     },
 778     serial => {
 779         type => 'string',
 780         format => 'urlencoded',
 781         format_description => 'serial',
 782         maxLength => 20*3, # *3 since it's %xx url enoded
 783         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 784         optional => 1,
 785     }
 786 );
 787
 788 my %rerror_fmt = (
 789     rerror => {
 790         type => 'string',
 791         enum => [qw(ignore report stop)],
 792         description => 'Read error action.',
 793         optional => 1,
 794     },
 795 );
 796
 797 my %iothread_fmt = ( iothread => {
 798         type => 'boolean',
 799         description => "Whether to use iothreads for this drive",
 800         optional => 1,
 801 });
 802
 803 my %model_fmt = (
 804     model => {
 805         type => 'string',
 806         format => 'urlencoded',
 807         format_description => 'model',
 808         maxLength => 40*3, # *3 since it's %xx url enoded
 809         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 810         optional => 1,
 811     },
 812 );
 813
 814 my %queues_fmt = (
 815     queues => {
 816         type => 'integer',
 817         description => "Number of queues.",
 818         minimum => 2,
 819         optional => 1
 820     }
 821 );
 822
 823 my $add_throttle_desc = sub {
 824     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 825     my $d = {
 826         type => $type,
 827         format_description => $unit,
 828         description => "Maximum $what in $longunit.",
 829         optional => 1,
 830     };
 831     $d->{minimum} = $minimum if defined($minimum);
 832     $drivedesc_base{$key} = $d;
 833 };
 834 # throughput: (leaky bucket)
 835 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 836 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 837 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 838 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 839 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 840 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 841 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 842 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 843 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 844
 845 # pools: (pool of IO before throttling starts taking effect)
 846 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 847 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 848 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 849 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 850 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 851 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 852
 853 # burst lengths
 854 $add_throttle_desc->('bps_max_length',  'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 855 $add_throttle_desc->('bps_rd_length',   'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 856 $add_throttle_desc->('bps_wr_length',   'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 857 $add_throttle_desc->('iops_max_length', 'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 858 $add_throttle_desc->('iops_rd_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 859 $add_throttle_desc->('iops_wr_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 860
 861 my $ide_fmt = {
 862     %drivedesc_base,
 863     %rerror_fmt,
 864     %model_fmt,
 865 };
 866 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 867
 868 my $idedesc = {
 869     optional => 1,
 870     type => 'string', format => $ide_fmt,
 871     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 872 };
 873 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 874
 875 my $scsi_fmt = {
 876     %drivedesc_base,
 877     %iothread_fmt,
 878     %queues_fmt,
 879 };
 880 my $scsidesc = {
 881     optional => 1,
 882     type => 'string', format => $scsi_fmt,
 883     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 884 };
 885 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 886
 887 my $sata_fmt = {
 888     %drivedesc_base,
 889     %rerror_fmt,
 890 };
 891 my $satadesc = {
 892     optional => 1,
 893     type => 'string', format => $sata_fmt,
 894     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 895 };
 896 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 897
 898 my $virtio_fmt = {
 899     %drivedesc_base,
 900     %iothread_fmt,
 901     %rerror_fmt,
 902 };
 903 my $virtiodesc = {
 904     optional => 1,
 905     type => 'string', format => $virtio_fmt,
 906     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 907 };
 908 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 909
 910 my $alldrive_fmt = {
 911     %drivedesc_base,
 912     %rerror_fmt,
 913     %iothread_fmt,
 914     %model_fmt,
 915     %queues_fmt,
 916 };
 917
 918 my $efidisk_fmt = {
 919     volume => { alias => 'file' },
 920     file => {
 921         type => 'string',
 922         format => 'pve-volume-id-or-qm-path',
 923         default_key => 1,
 924         format_description => 'volume',
 925         description => "The drive's backing volume.",
 926     },
 927     format => get_standard_option('pve-qm-image-format'),
 928     size => {
 929         type => 'string',
 930         format => 'disk-size',
 931         format_description => 'DiskSize',
 932         description => "Disk size. This is purely informational and has no effect.",
 933         optional => 1,
 934     },
 935 };
 936
 937 my $efidisk_desc = {
 938     optional => 1,
 939     type => 'string', format => $efidisk_fmt,
 940     description => "Configure a Disk for storing EFI vars",
 941 };
 942
 943 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 944
 945 my $usb_fmt = {
 946     host => {
 947         default_key => 1,
 948         type => 'string', format => 'pve-qm-usb-device',
 949         format_description => 'HOSTUSBDEVICE|spice',
 950         description => <<EODESCR,
 951 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 952
 953  'bus-port(.port)*' (decimal numbers) or
 954  'vendor_id:product_id' (hexadeciaml numbers) or
 955  'spice'
 956
 957 You can use the 'lsusb -t' command to list existing usb devices.
 958
 959 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 960
 961 The value 'spice' can be used to add a usb redirection devices for spice.
 962 EODESCR
 963     },
 964     usb3 => {
 965         optional => 1,
 966         type => 'boolean',
 967         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 968         default => 0,
 969     },
 970 };
 971
 972 my $usbdesc = {
 973     optional => 1,
 974     type => 'string', format => $usb_fmt,
 975     description => "Configure an USB device (n is 0 to 4).",
 976 };
 977 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 978
 979 # NOTE: the match-groups of this regex are used in parse_hostpci
 980 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 981 my $hostpci_fmt = {
 982     host => {
 983         default_key => 1,
 984         type => 'string',
 985         pattern => qr/$PCIRE(;$PCIRE)*/,
 986         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 987         description => <<EODESCR,
 988 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 989 of PCI virtual functions of the host. HOSTPCIID syntax is:
 990
 991 'bus:dev.func' (hexadecimal numbers)
 992
 993 You can us the 'lspci' command to list existing PCI devices.
 994 EODESCR
 995     },
 996     rombar => {
 997         type => 'boolean',
 998         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
 999         optional => 1,
1000         default => 1,
1001     },
1002     pcie => {
1003         type => 'boolean',
1004         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1005         optional => 1,
1006         default => 0,
1007     },
1008     'x-vga' => {
1009         type => 'boolean',
1010         description =>  "Enable vfio-vga device support.",
1011         optional => 1,
1012         default => 0,
1013     },
1014 };
1015 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1016
1017 my $hostpcidesc = {
1018         optional => 1,
1019         type => 'string', format => 'pve-qm-hostpci',
1020         description => "Map host PCI devices into guest.",
1021         verbose_description =>  <<EODESCR,
1022 Map host PCI devices into guest.
1023
1024 NOTE: This option allows direct access to host hardware. So it is no longer
1025 possible to migrate such machines - use with special care.
1026
1027 CAUTION: Experimental! User reported problems with this option.
1028 EODESCR
1029 };
1030 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1031
1032 my $serialdesc = {
1033         optional => 1,
1034         type => 'string',
1035         pattern => '(/dev/.+|socket)',
1036         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1037         verbose_description =>  <<EODESCR,
1038 Create a serial device inside the VM (n is 0 to 3), and pass through a
1039 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1040 host side (use 'qm terminal' to open a terminal connection).
1041
1042 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1043
1044 CAUTION: Experimental! User reported problems with this option.
1045 EODESCR
1046 };
1047
1048 my $paralleldesc= {
1049         optional => 1,
1050         type => 'string',
1051         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1052         description =>  "Map host parallel devices (n is 0 to 2).",
1053         verbose_description =>  <<EODESCR,
1054 Map host parallel devices (n is 0 to 2).
1055
1056 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1057
1058 CAUTION: Experimental! User reported problems with this option.
1059 EODESCR
1060 };
1061
1062 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1063     $confdesc->{"parallel$i"} = $paralleldesc;
1064 }
1065
1066 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1067     $confdesc->{"serial$i"} = $serialdesc;
1068 }
1069
1070 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1071     $confdesc->{"hostpci$i"} = $hostpcidesc;
1072 }
1073
1074 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1075     $drivename_hash->{"ide$i"} = 1;
1076     $confdesc->{"ide$i"} = $idedesc;
1077 }
1078
1079 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1080     $drivename_hash->{"sata$i"} = 1;
1081     $confdesc->{"sata$i"} = $satadesc;
1082 }
1083
1084 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1085     $drivename_hash->{"scsi$i"} = 1;
1086     $confdesc->{"scsi$i"} = $scsidesc ;
1087 }
1088
1089 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1090     $drivename_hash->{"virtio$i"} = 1;
1091     $confdesc->{"virtio$i"} = $virtiodesc;
1092 }
1093
1094 $drivename_hash->{efidisk0} = 1;
1095 $confdesc->{efidisk0} = $efidisk_desc;
1096
1097 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1098     $confdesc->{"usb$i"} = $usbdesc;
1099 }
1100
1101 my $unuseddesc = {
1102     optional => 1,
1103     type => 'string', format => 'pve-volume-id',
1104     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1105 };
1106
1107 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1108     $confdesc->{"unused$i"} = $unuseddesc;
1109 }
1110
1111 my $kvm_api_version = 0;
1112
1113 sub kvm_version {
1114
1115     return $kvm_api_version if $kvm_api_version;
1116
1117     my $fh = IO::File->new("</dev/kvm") ||
1118         return 0;
1119
1120     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1121         $kvm_api_version = $v;
1122     }
1123
1124     $fh->close();
1125
1126     return  $kvm_api_version;
1127 }
1128
1129 my $kvm_user_version;
1130
1131 sub kvm_user_version {
1132
1133     return $kvm_user_version if $kvm_user_version;
1134
1135     $kvm_user_version = 'unknown';
1136
1137     my $code = sub {
1138         my $line = shift;
1139         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1140             $kvm_user_version = $2;
1141         }
1142     };
1143
1144     eval { run_command("kvm -version", outfunc => $code); };
1145     warn $@ if $@;
1146
1147     return $kvm_user_version;
1148
1149 }
1150
1151 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1152
1153 sub valid_drive_names {
1154     # order is important - used to autoselect boot disk
1155     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1156             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1157             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1158             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1159             'efidisk0');
1160 }
1161
1162 sub is_valid_drivename {
1163     my $dev = shift;
1164
1165     return defined($drivename_hash->{$dev});
1166 }
1167
1168 sub option_exists {
1169     my $key = shift;
1170     return defined($confdesc->{$key});
1171 }
1172
1173 sub nic_models {
1174     return $nic_model_list;
1175 }
1176
1177 sub os_list_description {
1178
1179     return {
1180         other => 'Other',
1181         wxp => 'Windows XP',
1182         w2k => 'Windows 2000',
1183         w2k3 =>, 'Windows 2003',
1184         w2k8 => 'Windows 2008',
1185         wvista => 'Windows Vista',
1186         win7 => 'Windows 7',
1187         win8 => 'Windows 8/2012',
1188         win10 => 'Windows 10/2016',
1189         l24 => 'Linux 2.4',
1190         l26 => 'Linux 2.6',
1191     };
1192 }
1193
1194 my $cdrom_path;
1195
1196 sub get_cdrom_path {
1197
1198     return  $cdrom_path if $cdrom_path;
1199
1200     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1201     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1202     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1203 }
1204
1205 sub get_iso_path {
1206     my ($storecfg, $vmid, $cdrom) = @_;
1207
1208     if ($cdrom eq 'cdrom') {
1209         return get_cdrom_path();
1210     } elsif ($cdrom eq 'none') {
1211         return '';
1212     } elsif ($cdrom =~ m|^/|) {
1213         return $cdrom;
1214     } else {
1215         return PVE::Storage::path($storecfg, $cdrom);
1216     }
1217 }
1218
1219 # try to convert old style file names to volume IDs
1220 sub filename_to_volume_id {
1221     my ($vmid, $file, $media) = @_;
1222
1223     if (!($file eq 'none' || $file eq 'cdrom' ||
1224           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1225
1226         return undef if $file =~ m|/|;
1227
1228         if ($media && $media eq 'cdrom') {
1229             $file = "local:iso/$file";
1230         } else {
1231             $file = "local:$vmid/$file";
1232         }
1233     }
1234
1235     return $file;
1236 }
1237
1238 sub verify_media_type {
1239     my ($opt, $vtype, $media) = @_;
1240
1241     return if !$media;
1242
1243     my $etype;
1244     if ($media eq 'disk') {
1245         $etype = 'images';
1246     } elsif ($media eq 'cdrom') {
1247         $etype = 'iso';
1248     } else {
1249         die "internal error";
1250     }
1251
1252     return if ($vtype eq $etype);
1253
1254     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1255 }
1256
1257 sub cleanup_drive_path {
1258     my ($opt, $storecfg, $drive) = @_;
1259
1260     # try to convert filesystem paths to volume IDs
1261
1262     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1263         ($drive->{file} !~ m|^/dev/.+|) &&
1264         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1265         ($drive->{file} !~ m/^\d+$/)) {
1266         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1267         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1268         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1269         verify_media_type($opt, $vtype, $drive->{media});
1270         $drive->{file} = $volid;
1271     }
1272
1273     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1274 }
1275
1276 sub parse_hotplug_features {
1277     my ($data) = @_;
1278
1279     my $res = {};
1280
1281     return $res if $data eq '0';
1282
1283     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1284
1285     foreach my $feature (PVE::Tools::split_list($data)) {
1286         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1287             $res->{$1} = 1;
1288         } else {
1289             die "invalid hotplug feature '$feature'\n";
1290         }
1291     }
1292     return $res;
1293 }
1294
1295 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1296 sub pve_verify_hotplug_features {
1297     my ($value, $noerr) = @_;
1298
1299     return $value if parse_hotplug_features($value);
1300
1301     return undef if $noerr;
1302
1303     die "unable to parse hotplug option\n";
1304 }
1305
1306 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1307 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1308 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1309 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1310 #        [,iothread=on][,serial=serial][,model=model]
1311
1312 sub parse_drive {
1313     my ($key, $data) = @_;
1314
1315     my ($interface, $index);
1316
1317     if ($key =~ m/^([^\d]+)(\d+)$/) {
1318         $interface = $1;
1319         $index = $2;
1320     } else {
1321         return undef;
1322     }
1323
1324     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1325                                      : $confdesc->{$key}->{format};
1326     if (!$desc) {
1327         warn "invalid drive key: $key\n";
1328         return undef;
1329     }
1330     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1331     return undef if !$res;
1332     $res->{interface} = $interface;
1333     $res->{index} = $index;
1334
1335     my $error = 0;
1336     foreach my $opt (qw(bps bps_rd bps_wr)) {
1337         if (my $bps = defined(delete $res->{$opt})) {
1338             if (defined($res->{"m$opt"})) {
1339                 warn "both $opt and m$opt specified\n";
1340                 ++$error;
1341                 next;
1342             }
1343             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1344         }
1345     }
1346
1347     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1348     for my $requirement (
1349         [bps_max_length => 'mbps_max'],
1350         [bps_rd_max_length => 'mbps_rd_max'],
1351         [bps_wr_max_length => 'mbps_wr_max'],
1352         [iops_max_length => 'iops_max'],
1353         [iops_rd_max_length => 'iops_rd_max'],
1354         [iops_wr_max_length => 'iops_wr_max']) {
1355         my ($option, $requires) = @$requirement;
1356         if ($res->{$option} && !$res->{$requires}) {
1357             warn "$option requires $requires\n";
1358             ++$error;
1359         }
1360     }
1361
1362     return undef if $error;
1363
1364     return undef if $res->{mbps_rd} && $res->{mbps};
1365     return undef if $res->{mbps_wr} && $res->{mbps};
1366     return undef if $res->{iops_rd} && $res->{iops};
1367     return undef if $res->{iops_wr} && $res->{iops};
1368
1369     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1370         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1371         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1372         return undef if $res->{interface} eq 'virtio';
1373     }
1374
1375     if (my $size = $res->{size}) {
1376         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1377     }
1378
1379     return $res;
1380 }
1381
1382 sub print_drive {
1383     my ($vmid, $drive) = @_;
1384     my $data = { %$drive };
1385     delete $data->{$_} for qw(index interface);
1386     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1387 }
1388
1389 sub scsi_inquiry {
1390     my($fh, $noerr) = @_;
1391
1392     my $SG_IO = 0x2285;
1393     my $SG_GET_VERSION_NUM = 0x2282;
1394
1395     my $versionbuf = "\x00" x 8;
1396     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1397     if (!$ret) {
1398         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1399         return undef;
1400     }
1401     my $version = unpack("I", $versionbuf);
1402     if ($version < 30000) {
1403         die "scsi generic interface too old\n"  if !$noerr;
1404         return undef;
1405     }
1406
1407     my $buf = "\x00" x 36;
1408     my $sensebuf = "\x00" x 8;
1409     my $cmd = pack("C x3 C x1", 0x12, 36);
1410
1411     # see /usr/include/scsi/sg.h
1412     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1413
1414     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1415                       length($sensebuf), 0, length($buf), $buf,
1416                       $cmd, $sensebuf, 6000);
1417
1418     $ret = ioctl($fh, $SG_IO, $packet);
1419     if (!$ret) {
1420         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1421         return undef;
1422     }
1423
1424     my @res = unpack($sg_io_hdr_t, $packet);
1425     if ($res[17] || $res[18]) {
1426         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1427         return undef;
1428     }
1429
1430     my $res = {};
1431     (my $byte0, my $byte1, $res->{vendor},
1432      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1433
1434     $res->{removable} = $byte1 & 128 ? 1 : 0;
1435     $res->{type} = $byte0 & 31;
1436
1437     return $res;
1438 }
1439
1440 sub path_is_scsi {
1441     my ($path) = @_;
1442
1443     my $fh = IO::File->new("+<$path") || return undef;
1444     my $res = scsi_inquiry($fh, 1);
1445     close($fh);
1446
1447     return $res;
1448 }
1449
1450 sub machine_type_is_q35 {
1451     my ($conf) = @_;
1452
1453     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1454 }
1455
1456 sub print_tabletdevice_full {
1457     my ($conf) = @_;
1458
1459     my $q35 = machine_type_is_q35($conf);
1460
1461     # we use uhci for old VMs because tablet driver was buggy in older qemu
1462     my $usbbus = $q35 ? "ehci" : "uhci";
1463
1464     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1465 }
1466
1467 sub print_drivedevice_full {
1468     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1469
1470     my $device = '';
1471     my $maxdev = 0;
1472
1473     if ($drive->{interface} eq 'virtio') {
1474         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1475         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1476         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1477     } elsif ($drive->{interface} eq 'scsi') {
1478
1479         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1480         my $unit = $drive->{index} % $maxdev;
1481         my $devicetype = 'hd';
1482         my $path = '';
1483         if (drive_is_cdrom($drive)) {
1484             $devicetype = 'cd';
1485         } else {
1486             if ($drive->{file} =~ m|^/|) {
1487                 $path = $drive->{file};
1488                 if (my $info = path_is_scsi($path)) {
1489                     if ($info->{type} == 0) {
1490                         $devicetype = 'block';
1491                     } elsif ($info->{type} == 1) { # tape
1492                         $devicetype = 'generic';
1493                     }
1494                 }
1495             } else {
1496                  $path = PVE::Storage::path($storecfg, $drive->{file});
1497             }
1498
1499             if($path =~ m/^iscsi\:\/\//){
1500                 $devicetype = 'generic';
1501             }
1502         }
1503
1504         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1505             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1506         } else {
1507             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1508         }
1509
1510     } elsif ($drive->{interface} eq 'ide'){
1511         $maxdev = 2;
1512         my $controller = int($drive->{index} / $maxdev);
1513         my $unit = $drive->{index} % $maxdev;
1514         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1515
1516         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1517         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1518             $model = URI::Escape::uri_unescape($model);
1519             $device .= ",model=$model";
1520         }
1521     } elsif ($drive->{interface} eq 'sata'){
1522         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1523         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1524         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1525     } elsif ($drive->{interface} eq 'usb') {
1526         die "implement me";
1527         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1528     } else {
1529         die "unsupported interface type";
1530     }
1531
1532     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1533
1534     return $device;
1535 }
1536
1537 sub get_initiator_name {
1538     my $initiator;
1539
1540     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1541     while (defined(my $line = <$fh>)) {
1542         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1543         $initiator = $1;
1544         last;
1545     }
1546     $fh->close();
1547
1548     return $initiator;
1549 }
1550
1551 sub print_drive_full {
1552     my ($storecfg, $vmid, $drive) = @_;
1553
1554     my $path;
1555     my $volid = $drive->{file};
1556     my $format;
1557
1558     if (drive_is_cdrom($drive)) {
1559         $path = get_iso_path($storecfg, $vmid, $volid);
1560     } else {
1561         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1562         if ($storeid) {
1563             $path = PVE::Storage::path($storecfg, $volid);
1564             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1565             $format = qemu_img_format($scfg, $volname);
1566         } else {
1567             $path = $volid;
1568             $format = "raw";
1569         }
1570    }
1571
1572     my $opts = '';
1573     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1574     foreach my $o (@qemu_drive_options) {
1575         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1576     }
1577     if (my $serial = $drive->{serial}) {
1578         $serial = URI::Escape::uri_unescape($serial);
1579         $opts .= ",serial=$serial";
1580     }
1581
1582     $opts .= ",format=$format" if $format && !$drive->{format};
1583
1584     foreach my $o (qw(bps bps_rd bps_wr)) {
1585         my $v = $drive->{"m$o"};
1586         $opts .= ",$o=" . int($v*1024*1024) if $v;
1587     }
1588
1589     my $cache_direct = 0;
1590
1591     if (my $cache = $drive->{cache}) {
1592         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1593     } elsif (!drive_is_cdrom($drive)) {
1594         $opts .= ",cache=none";
1595         $cache_direct = 1;
1596     }
1597
1598     # aio native works only with O_DIRECT
1599     if (!$drive->{aio}) {
1600         if($cache_direct) {
1601             $opts .= ",aio=native";
1602         } else {
1603             $opts .= ",aio=threads";
1604         }
1605     }
1606
1607     if (!drive_is_cdrom($drive)) {
1608         my $detectzeroes;
1609         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1610             $detectzeroes = 'off';
1611         } elsif ($drive->{discard}) {
1612             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1613         } else {
1614             # This used to be our default with discard not being specified:
1615             $detectzeroes = 'on';
1616         }
1617         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1618     }
1619
1620     my $pathinfo = $path ? "file=$path," : '';
1621
1622     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1623 }
1624
1625 sub print_netdevice_full {
1626     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1627
1628     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1629
1630     my $device = $net->{model};
1631     if ($net->{model} eq 'virtio') {
1632          $device = 'virtio-net-pci';
1633      };
1634
1635     my $pciaddr = print_pci_addr("$netid", $bridges);
1636     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1637     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1638         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1639         my $vectors = $net->{queues} * 2 + 2;
1640         $tmpstr .= ",vectors=$vectors,mq=on";
1641     }
1642     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1643
1644     if ($use_old_bios_files) {
1645         my $romfile;
1646         if ($device eq 'virtio-net-pci') {
1647             $romfile = 'pxe-virtio.rom';
1648         } elsif ($device eq 'e1000') {
1649             $romfile = 'pxe-e1000.rom';
1650         } elsif ($device eq 'ne2k') {
1651             $romfile = 'pxe-ne2k_pci.rom';
1652         } elsif ($device eq 'pcnet') {
1653             $romfile = 'pxe-pcnet.rom';
1654         } elsif ($device eq 'rtl8139') {
1655             $romfile = 'pxe-rtl8139.rom';
1656         }
1657         $tmpstr .= ",romfile=$romfile" if $romfile;
1658     }
1659
1660     return $tmpstr;
1661 }
1662
1663 sub print_netdev_full {
1664     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1665
1666     my $i = '';
1667     if ($netid =~ m/^net(\d+)$/) {
1668         $i = int($1);
1669     }
1670
1671     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1672
1673     my $ifname = "tap${vmid}i$i";
1674
1675     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1676     die "interface name '$ifname' is too long (max 15 character)\n"
1677         if length($ifname) >= 16;
1678
1679     my $vhostparam = '';
1680     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1681
1682     my $vmname = $conf->{name} || "vm$vmid";
1683
1684     my $netdev = "";
1685     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1686
1687     if ($net->{bridge}) {
1688         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1689     } else {
1690         $netdev = "type=user,id=$netid,hostname=$vmname";
1691     }
1692
1693     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1694
1695     return $netdev;
1696 }
1697
1698
1699 sub print_cpu_device {
1700     my ($conf, $id) = @_;
1701
1702     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
1703     my $cpu = $nokvm ? "qemu64" : "kvm64";
1704     if (my $cputype = $conf->{cpu}) {
1705         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1706             or die "Cannot parse cpu description: $cputype\n";
1707         $cpu = $cpuconf->{cputype};
1708     }
1709
1710     my $sockets = 1;
1711     $sockets = $conf->{sockets} if  $conf->{sockets};
1712     my $cores = $conf->{cores} || 1;
1713
1714     my $current_core = ($id - 1) % $cores;
1715     my $current_socket = int(($id - $current_core)/$cores);
1716
1717     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1718 }
1719
1720 sub drive_is_cdrom {
1721     my ($drive) = @_;
1722
1723     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1724
1725 }
1726
1727 sub parse_number_sets {
1728     my ($set) = @_;
1729     my $res = [];
1730     foreach my $part (split(/;/, $set)) {
1731         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1732             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1733             push @$res, [ $1, $2 ];
1734         } else {
1735             die "invalid range: $part\n";
1736         }
1737     }
1738     return $res;
1739 }
1740
1741 sub parse_numa {
1742     my ($data) = @_;
1743
1744     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1745     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1746     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1747     return $res;
1748 }
1749
1750 sub parse_hostpci {
1751     my ($value) = @_;
1752
1753     return undef if !$value;
1754
1755     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1756
1757     my @idlist = split(/;/, $res->{host});
1758     delete $res->{host};
1759     foreach my $id (@idlist) {
1760         if ($id =~ /^$PCIRE$/) {
1761             if (defined($2)) {
1762                 push @{$res->{pciid}}, { id => $1, function => $2 };
1763             } else {
1764                 my $pcidevices = lspci($1);
1765                 $res->{pciid} = $pcidevices->{$1};
1766             }
1767         } else {
1768             # should have been caught by parse_property_string already
1769             die "failed to parse PCI id: $id\n";
1770         }
1771     }
1772     return $res;
1773 }
1774
1775 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1776 sub parse_net {
1777     my ($data) = @_;
1778
1779     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1780     if ($@) {
1781         warn $@;
1782         return undef;
1783     }
1784     if (!defined($res->{macaddr})) {
1785         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1786         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1787     }
1788     return $res;
1789 }
1790
1791 sub print_net {
1792     my $net = shift;
1793
1794     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1795 }
1796
1797 sub add_random_macs {
1798     my ($settings) = @_;
1799
1800     foreach my $opt (keys %$settings) {
1801         next if $opt !~ m/^net(\d+)$/;
1802         my $net = parse_net($settings->{$opt});
1803         next if !$net;
1804         $settings->{$opt} = print_net($net);
1805     }
1806 }
1807
1808 sub vm_is_volid_owner {
1809     my ($storecfg, $vmid, $volid) = @_;
1810
1811     if ($volid !~  m|^/|) {
1812         my ($path, $owner);
1813         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1814         if ($owner && ($owner == $vmid)) {
1815             return 1;
1816         }
1817     }
1818
1819     return undef;
1820 }
1821
1822 sub split_flagged_list {
1823     my $text = shift || '';
1824     $text =~ s/[,;]/ /g;
1825     $text =~ s/^\s+//;
1826     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1827 }
1828
1829 sub join_flagged_list {
1830     my ($how, $lst) = @_;
1831     join $how, map { $lst->{$_} . $_ } keys %$lst;
1832 }
1833
1834 sub vmconfig_delete_pending_option {
1835     my ($conf, $key, $force) = @_;
1836
1837     delete $conf->{pending}->{$key};
1838     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1839     $pending_delete_hash->{$key} = $force ? '!' : '';
1840     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1841 }
1842
1843 sub vmconfig_undelete_pending_option {
1844     my ($conf, $key) = @_;
1845
1846     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1847     delete $pending_delete_hash->{$key};
1848
1849     if (%$pending_delete_hash) {
1850         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1851     } else {
1852         delete $conf->{pending}->{delete};
1853     }
1854 }
1855
1856 sub vmconfig_register_unused_drive {
1857     my ($storecfg, $vmid, $conf, $drive) = @_;
1858
1859     if (!drive_is_cdrom($drive)) {
1860         my $volid = $drive->{file};
1861         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1862             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1863         }
1864     }
1865 }
1866
1867 sub vmconfig_cleanup_pending {
1868     my ($conf) = @_;
1869
1870     # remove pending changes when nothing changed
1871     my $changes;
1872     foreach my $opt (keys %{$conf->{pending}}) {
1873         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1874             $changes = 1;
1875             delete $conf->{pending}->{$opt};
1876         }
1877     }
1878
1879     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1880     my $pending_delete_hash = {};
1881     while (my ($opt, $force) = each %$current_delete_hash) {
1882         if (defined($conf->{$opt})) {
1883             $pending_delete_hash->{$opt} = $force;
1884         } else {
1885             $changes = 1;
1886         }
1887     }
1888
1889     if (%$pending_delete_hash) {
1890         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1891     } else {
1892         delete $conf->{pending}->{delete};
1893     }
1894
1895     return $changes;
1896 }
1897
1898 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1899 my $smbios1_fmt = {
1900     uuid => {
1901         type => 'string',
1902         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1903         format_description => 'UUID',
1904         description => "Set SMBIOS1 UUID.",
1905         optional => 1,
1906     },
1907     version => {
1908         type => 'string',
1909         pattern => '\S+',
1910         format_description => 'string',
1911         description => "Set SMBIOS1 version.",
1912         optional => 1,
1913     },
1914     serial => {
1915         type => 'string',
1916         pattern => '\S+',
1917         format_description => 'string',
1918         description => "Set SMBIOS1 serial number.",
1919         optional => 1,
1920     },
1921     manufacturer => {
1922         type => 'string',
1923         pattern => '\S+',
1924         format_description => 'string',
1925         description => "Set SMBIOS1 manufacturer.",
1926         optional => 1,
1927     },
1928     product => {
1929         type => 'string',
1930         pattern => '\S+',
1931         format_description => 'string',
1932         description => "Set SMBIOS1 product ID.",
1933         optional => 1,
1934     },
1935     sku => {
1936         type => 'string',
1937         pattern => '\S+',
1938         format_description => 'string',
1939         description => "Set SMBIOS1 SKU string.",
1940         optional => 1,
1941     },
1942     family => {
1943         type => 'string',
1944         pattern => '\S+',
1945         format_description => 'string',
1946         description => "Set SMBIOS1 family string.",
1947         optional => 1,
1948     },
1949 };
1950
1951 sub parse_smbios1 {
1952     my ($data) = @_;
1953
1954     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1955     warn $@ if $@;
1956     return $res;
1957 }
1958
1959 sub print_smbios1 {
1960     my ($smbios1) = @_;
1961     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1962 }
1963
1964 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1965
1966 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1967 sub verify_bootdisk {
1968     my ($value, $noerr) = @_;
1969
1970     return $value if is_valid_drivename($value);
1971
1972     return undef if $noerr;
1973
1974     die "invalid boot disk '$value'\n";
1975 }
1976
1977 sub parse_watchdog {
1978     my ($value) = @_;
1979
1980     return undef if !$value;
1981
1982     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1983     warn $@ if $@;
1984     return $res;
1985 }
1986
1987 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1988 sub verify_usb_device {
1989     my ($value, $noerr) = @_;
1990
1991     return $value if parse_usb_device($value);
1992
1993     return undef if $noerr;
1994
1995     die "unable to parse usb device\n";
1996 }
1997
1998 # add JSON properties for create and set function
1999 sub json_config_properties {
2000     my $prop = shift;
2001
2002     foreach my $opt (keys %$confdesc) {
2003         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2004         $prop->{$opt} = $confdesc->{$opt};
2005     }
2006
2007     return $prop;
2008 }
2009
2010 sub check_type {
2011     my ($key, $value) = @_;
2012
2013     die "unknown setting '$key'\n" if !$confdesc->{$key};
2014
2015     my $type = $confdesc->{$key}->{type};
2016
2017     if (!defined($value)) {
2018         die "got undefined value\n";
2019     }
2020
2021     if ($value =~ m/[\n\r]/) {
2022         die "property contains a line feed\n";
2023     }
2024
2025     if ($type eq 'boolean') {
2026         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2027         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2028         die "type check ('boolean') failed - got '$value'\n";
2029     } elsif ($type eq 'integer') {
2030         return int($1) if $value =~ m/^(\d+)$/;
2031         die "type check ('integer') failed - got '$value'\n";
2032     } elsif ($type eq 'number') {
2033         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2034         die "type check ('number') failed - got '$value'\n";
2035     } elsif ($type eq 'string') {
2036         if (my $fmt = $confdesc->{$key}->{format}) {
2037             PVE::JSONSchema::check_format($fmt, $value);
2038             return $value;
2039         }
2040         $value =~ s/^\"(.*)\"$/$1/;
2041         return $value;
2042     } else {
2043         die "internal error"
2044     }
2045 }
2046
2047 sub check_iommu_support{
2048     #fixme : need to check IOMMU support
2049     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2050
2051     my $iommu=1;
2052     return $iommu;
2053
2054 }
2055
2056 sub touch_config {
2057     my ($vmid) = @_;
2058
2059     my $conf = PVE::QemuConfig->config_file($vmid);
2060     utime undef, undef, $conf;
2061 }
2062
2063 sub destroy_vm {
2064     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2065
2066     my $conffile = PVE::QemuConfig->config_file($vmid);
2067
2068     my $conf = PVE::QemuConfig->load_config($vmid);
2069
2070     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2071
2072     # only remove disks owned by this VM
2073     foreach_drive($conf, sub {
2074         my ($ds, $drive) = @_;
2075
2076         return if drive_is_cdrom($drive);
2077
2078         my $volid = $drive->{file};
2079
2080         return if !$volid || $volid =~ m|^/|;
2081
2082         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2083         return if !$path || !$owner || ($owner != $vmid);
2084
2085         PVE::Storage::vdisk_free($storecfg, $volid);
2086     });
2087
2088     if ($keep_empty_config) {
2089         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2090     } else {
2091         unlink $conffile;
2092     }
2093
2094     # also remove unused disk
2095     eval {
2096         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2097
2098         eval {
2099             PVE::Storage::foreach_volid($dl, sub {
2100                 my ($volid, $sid, $volname, $d) = @_;
2101                 PVE::Storage::vdisk_free($storecfg, $volid);
2102             });
2103         };
2104         warn $@ if $@;
2105
2106     };
2107     warn $@ if $@;
2108 }
2109
2110 sub parse_vm_config {
2111     my ($filename, $raw) = @_;
2112
2113     return undef if !defined($raw);
2114
2115     my $res = {
2116         digest => Digest::SHA::sha1_hex($raw),
2117         snapshots => {},
2118         pending => {},
2119     };
2120
2121     $filename =~ m|/qemu-server/(\d+)\.conf$|
2122         || die "got strange filename '$filename'";
2123
2124     my $vmid = $1;
2125
2126     my $conf = $res;
2127     my $descr;
2128     my $section = '';
2129
2130     my @lines = split(/\n/, $raw);
2131     foreach my $line (@lines) {
2132         next if $line =~ m/^\s*$/;
2133
2134         if ($line =~ m/^\[PENDING\]\s*$/i) {
2135             $section = 'pending';
2136             if (defined($descr)) {
2137                 $descr =~ s/\s+$//;
2138                 $conf->{description} = $descr;
2139             }
2140             $descr = undef;
2141             $conf = $res->{$section} = {};
2142             next;
2143
2144         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2145             $section = $1;
2146             if (defined($descr)) {
2147                 $descr =~ s/\s+$//;
2148                 $conf->{description} = $descr;
2149             }
2150             $descr = undef;
2151             $conf = $res->{snapshots}->{$section} = {};
2152             next;
2153         }
2154
2155         if ($line =~ m/^\#(.*)\s*$/) {
2156             $descr = '' if !defined($descr);
2157             $descr .= PVE::Tools::decode_text($1) . "\n";
2158             next;
2159         }
2160
2161         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2162             $descr = '' if !defined($descr);
2163             $descr .= PVE::Tools::decode_text($2);
2164         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2165             $conf->{snapstate} = $1;
2166         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2167             my $key = $1;
2168             my $value = $2;
2169             $conf->{$key} = $value;
2170         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2171             my $value = $1;
2172             if ($section eq 'pending') {
2173                 $conf->{delete} = $value; # we parse this later
2174             } else {
2175                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2176             }
2177         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2178             my $key = $1;
2179             my $value = $2;
2180             eval { $value = check_type($key, $value); };
2181             if ($@) {
2182                 warn "vm $vmid - unable to parse value of '$key' - $@";
2183             } else {
2184                 $key = 'ide2' if $key eq 'cdrom';
2185                 my $fmt = $confdesc->{$key}->{format};
2186                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2187                     my $v = parse_drive($key, $value);
2188                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2189                         $v->{file} = $volid;
2190                         $value = print_drive($vmid, $v);
2191                     } else {
2192                         warn "vm $vmid - unable to parse value of '$key'\n";
2193                         next;
2194                     }
2195                 }
2196
2197                 $conf->{$key} = $value;
2198             }
2199         }
2200     }
2201
2202     if (defined($descr)) {
2203         $descr =~ s/\s+$//;
2204         $conf->{description} = $descr;
2205     }
2206     delete $res->{snapstate}; # just to be sure
2207
2208     return $res;
2209 }
2210
2211 sub write_vm_config {
2212     my ($filename, $conf) = @_;
2213
2214     delete $conf->{snapstate}; # just to be sure
2215
2216     if ($conf->{cdrom}) {
2217         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2218         $conf->{ide2} = $conf->{cdrom};
2219         delete $conf->{cdrom};
2220     }
2221
2222     # we do not use 'smp' any longer
2223     if ($conf->{sockets}) {
2224         delete $conf->{smp};
2225     } elsif ($conf->{smp}) {
2226         $conf->{sockets} = $conf->{smp};
2227         delete $conf->{cores};
2228         delete $conf->{smp};
2229     }
2230
2231     my $used_volids = {};
2232
2233     my $cleanup_config = sub {
2234         my ($cref, $pending, $snapname) = @_;
2235
2236         foreach my $key (keys %$cref) {
2237             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2238                 $key eq 'snapstate' || $key eq 'pending';
2239             my $value = $cref->{$key};
2240             if ($key eq 'delete') {
2241                 die "propertry 'delete' is only allowed in [PENDING]\n"
2242                     if !$pending;
2243                 # fixme: check syntax?
2244                 next;
2245             }
2246             eval { $value = check_type($key, $value); };
2247             die "unable to parse value of '$key' - $@" if $@;
2248
2249             $cref->{$key} = $value;
2250
2251             if (!$snapname && is_valid_drivename($key)) {
2252                 my $drive = parse_drive($key, $value);
2253                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2254             }
2255         }
2256     };
2257
2258     &$cleanup_config($conf);
2259
2260     &$cleanup_config($conf->{pending}, 1);
2261
2262     foreach my $snapname (keys %{$conf->{snapshots}}) {
2263         die "internal error" if $snapname eq 'pending';
2264         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2265     }
2266
2267     # remove 'unusedX' settings if we re-add a volume
2268     foreach my $key (keys %$conf) {
2269         my $value = $conf->{$key};
2270         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2271             delete $conf->{$key};
2272         }
2273     }
2274
2275     my $generate_raw_config = sub {
2276         my ($conf, $pending) = @_;
2277
2278         my $raw = '';
2279
2280         # add description as comment to top of file
2281         if (defined(my $descr = $conf->{description})) {
2282             if ($descr) {
2283                 foreach my $cl (split(/\n/, $descr)) {
2284                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2285                 }
2286             } else {
2287                 $raw .= "#\n" if $pending;
2288             }
2289         }
2290
2291         foreach my $key (sort keys %$conf) {
2292             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2293             $raw .= "$key: $conf->{$key}\n";
2294         }
2295         return $raw;
2296     };
2297
2298     my $raw = &$generate_raw_config($conf);
2299
2300     if (scalar(keys %{$conf->{pending}})){
2301         $raw .= "\n[PENDING]\n";
2302         $raw .= &$generate_raw_config($conf->{pending}, 1);
2303     }
2304
2305     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2306         $raw .= "\n[$snapname]\n";
2307         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2308     }
2309
2310     return $raw;
2311 }
2312
2313 sub load_defaults {
2314
2315     my $res = {};
2316
2317     # we use static defaults from our JSON schema configuration
2318     foreach my $key (keys %$confdesc) {
2319         if (defined(my $default = $confdesc->{$key}->{default})) {
2320             $res->{$key} = $default;
2321         }
2322     }
2323
2324     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2325     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2326
2327     return $res;
2328 }
2329
2330 sub config_list {
2331     my $vmlist = PVE::Cluster::get_vmlist();
2332     my $res = {};
2333     return $res if !$vmlist || !$vmlist->{ids};
2334     my $ids = $vmlist->{ids};
2335
2336     foreach my $vmid (keys %$ids) {
2337         my $d = $ids->{$vmid};
2338         next if !$d->{node} || $d->{node} ne $nodename;
2339         next if !$d->{type} || $d->{type} ne 'qemu';
2340         $res->{$vmid}->{exists} = 1;
2341     }
2342     return $res;
2343 }
2344
2345 # test if VM uses local resources (to prevent migration)
2346 sub check_local_resources {
2347     my ($conf, $noerr) = @_;
2348
2349     my $loc_res = 0;
2350
2351     $loc_res = 1 if $conf->{hostusb}; # old syntax
2352     $loc_res = 1 if $conf->{hostpci}; # old syntax
2353
2354     foreach my $k (keys %$conf) {
2355         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2356         # sockets are safe: they will recreated be on the target side post-migrate
2357         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2358         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2359     }
2360
2361     die "VM uses local resources\n" if $loc_res && !$noerr;
2362
2363     return $loc_res;
2364 }
2365
2366 # check if used storages are available on all nodes (use by migrate)
2367 sub check_storage_availability {
2368     my ($storecfg, $conf, $node) = @_;
2369
2370     foreach_drive($conf, sub {
2371         my ($ds, $drive) = @_;
2372
2373         my $volid = $drive->{file};
2374         return if !$volid;
2375
2376         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2377         return if !$sid;
2378
2379         # check if storage is available on both nodes
2380         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2381         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2382    });
2383 }
2384
2385 # list nodes where all VM images are available (used by has_feature API)
2386 sub shared_nodes {
2387     my ($conf, $storecfg) = @_;
2388
2389     my $nodelist = PVE::Cluster::get_nodelist();
2390     my $nodehash = { map { $_ => 1 } @$nodelist };
2391     my $nodename = PVE::INotify::nodename();
2392
2393     foreach_drive($conf, sub {
2394         my ($ds, $drive) = @_;
2395
2396         my $volid = $drive->{file};
2397         return if !$volid;
2398
2399         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2400         if ($storeid) {
2401             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2402             if ($scfg->{disable}) {
2403                 $nodehash = {};
2404             } elsif (my $avail = $scfg->{nodes}) {
2405                 foreach my $node (keys %$nodehash) {
2406                     delete $nodehash->{$node} if !$avail->{$node};
2407                 }
2408             } elsif (!$scfg->{shared}) {
2409                 foreach my $node (keys %$nodehash) {
2410                     delete $nodehash->{$node} if $node ne $nodename
2411                 }
2412             }
2413         }
2414     });
2415
2416     return $nodehash
2417 }
2418
2419 sub check_cmdline {
2420     my ($pidfile, $pid) = @_;
2421
2422     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2423     if (defined($fh)) {
2424         my $line = <$fh>;
2425         $fh->close;
2426         return undef if !$line;
2427         my @param = split(/\0/, $line);
2428
2429         my $cmd = $param[0];
2430         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2431
2432         for (my $i = 0; $i < scalar (@param); $i++) {
2433             my $p = $param[$i];
2434             next if !$p;
2435             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2436                 my $p = $param[$i+1];
2437                 return 1 if $p && ($p eq $pidfile);
2438                 return undef;
2439             }
2440         }
2441     }
2442     return undef;
2443 }
2444
2445 sub check_running {
2446     my ($vmid, $nocheck, $node) = @_;
2447
2448     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2449
2450     die "unable to find configuration file for VM $vmid - no such machine\n"
2451         if !$nocheck && ! -f $filename;
2452
2453     my $pidfile = pidfile_name($vmid);
2454
2455     if (my $fd = IO::File->new("<$pidfile")) {
2456         my $st = stat($fd);
2457         my $line = <$fd>;
2458         close($fd);
2459
2460         my $mtime = $st->mtime;
2461         if ($mtime > time()) {
2462             warn "file '$filename' modified in future\n";
2463         }
2464
2465         if ($line =~ m/^(\d+)$/) {
2466             my $pid = $1;
2467             if (check_cmdline($pidfile, $pid)) {
2468                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2469                     return $pid;
2470                 }
2471             }
2472         }
2473     }
2474
2475     return undef;
2476 }
2477
2478 sub vzlist {
2479
2480     my $vzlist = config_list();
2481
2482     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2483
2484     while (defined(my $de = $fd->read)) {
2485         next if $de !~ m/^(\d+)\.pid$/;
2486         my $vmid = $1;
2487         next if !defined($vzlist->{$vmid});
2488         if (my $pid = check_running($vmid)) {
2489             $vzlist->{$vmid}->{pid} = $pid;
2490         }
2491     }
2492
2493     return $vzlist;
2494 }
2495
2496 sub disksize {
2497     my ($storecfg, $conf) = @_;
2498
2499     my $bootdisk = $conf->{bootdisk};
2500     return undef if !$bootdisk;
2501     return undef if !is_valid_drivename($bootdisk);
2502
2503     return undef if !$conf->{$bootdisk};
2504
2505     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2506     return undef if !defined($drive);
2507
2508     return undef if drive_is_cdrom($drive);
2509
2510     my $volid = $drive->{file};
2511     return undef if !$volid;
2512
2513     return $drive->{size};
2514 }
2515
2516 my $last_proc_pid_stat;
2517
2518 # get VM status information
2519 # This must be fast and should not block ($full == false)
2520 # We only query KVM using QMP if $full == true (this can be slow)
2521 sub vmstatus {
2522     my ($opt_vmid, $full) = @_;
2523
2524     my $res = {};
2525
2526     my $storecfg = PVE::Storage::config();
2527
2528     my $list = vzlist();
2529     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2530
2531     my $cpucount = $cpuinfo->{cpus} || 1;
2532
2533     foreach my $vmid (keys %$list) {
2534         next if $opt_vmid && ($vmid ne $opt_vmid);
2535
2536         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2537         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2538
2539         my $d = {};
2540         $d->{pid} = $list->{$vmid}->{pid};
2541
2542         # fixme: better status?
2543         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2544
2545         my $size = disksize($storecfg, $conf);
2546         if (defined($size)) {
2547             $d->{disk} = 0; # no info available
2548             $d->{maxdisk} = $size;
2549         } else {
2550             $d->{disk} = 0;
2551             $d->{maxdisk} = 0;
2552         }
2553
2554         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2555         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2556         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2557
2558         $d->{name} = $conf->{name} || "VM $vmid";
2559         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2560
2561         if ($conf->{balloon}) {
2562             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2563             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2564         }
2565
2566         $d->{uptime} = 0;
2567         $d->{cpu} = 0;
2568         $d->{mem} = 0;
2569
2570         $d->{netout} = 0;
2571         $d->{netin} = 0;
2572
2573         $d->{diskread} = 0;
2574         $d->{diskwrite} = 0;
2575
2576         $d->{template} = PVE::QemuConfig->is_template($conf);
2577
2578         $res->{$vmid} = $d;
2579     }
2580
2581     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2582     foreach my $dev (keys %$netdev) {
2583         next if $dev !~ m/^tap([1-9]\d*)i/;
2584         my $vmid = $1;
2585         my $d = $res->{$vmid};
2586         next if !$d;
2587
2588         $d->{netout} += $netdev->{$dev}->{receive};
2589         $d->{netin} += $netdev->{$dev}->{transmit};
2590
2591         if ($full) {
2592             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2593             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2594         }
2595
2596     }
2597
2598     my $ctime = gettimeofday;
2599
2600     foreach my $vmid (keys %$list) {
2601
2602         my $d = $res->{$vmid};
2603         my $pid = $d->{pid};
2604         next if !$pid;
2605
2606         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2607         next if !$pstat; # not running
2608
2609         my $used = $pstat->{utime} + $pstat->{stime};
2610
2611         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2612
2613         if ($pstat->{vsize}) {
2614             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2615         }
2616
2617         my $old = $last_proc_pid_stat->{$pid};
2618         if (!$old) {
2619             $last_proc_pid_stat->{$pid} = {
2620                 time => $ctime,
2621                 used => $used,
2622                 cpu => 0,
2623             };
2624             next;
2625         }
2626
2627         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2628
2629         if ($dtime > 1000) {
2630             my $dutime = $used -  $old->{used};
2631
2632             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2633             $last_proc_pid_stat->{$pid} = {
2634                 time => $ctime,
2635                 used => $used,
2636                 cpu => $d->{cpu},
2637             };
2638         } else {
2639             $d->{cpu} = $old->{cpu};
2640         }
2641     }
2642
2643     return $res if !$full;
2644
2645     my $qmpclient = PVE::QMPClient->new();
2646
2647     my $ballooncb = sub {
2648         my ($vmid, $resp) = @_;
2649
2650         my $info = $resp->{'return'};
2651         return if !$info->{max_mem};
2652
2653         my $d = $res->{$vmid};
2654
2655         # use memory assigned to VM
2656         $d->{maxmem} = $info->{max_mem};
2657         $d->{balloon} = $info->{actual};
2658
2659         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2660             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2661             $d->{freemem} = $info->{free_mem};
2662         }
2663
2664         $d->{ballooninfo} = $info;
2665     };
2666
2667     my $blockstatscb = sub {
2668         my ($vmid, $resp) = @_;
2669         my $data = $resp->{'return'} || [];
2670         my $totalrdbytes = 0;
2671         my $totalwrbytes = 0;
2672
2673         for my $blockstat (@$data) {
2674             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2675             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2676
2677             $blockstat->{device} =~ s/drive-//;
2678             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2679         }
2680         $res->{$vmid}->{diskread} = $totalrdbytes;
2681         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2682     };
2683
2684     my $statuscb = sub {
2685         my ($vmid, $resp) = @_;
2686
2687         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2688         # this fails if ballon driver is not loaded, so this must be
2689         # the last commnand (following command are aborted if this fails).
2690         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2691
2692         my $status = 'unknown';
2693         if (!defined($status = $resp->{'return'}->{status})) {
2694             warn "unable to get VM status\n";
2695             return;
2696         }
2697
2698         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2699     };
2700
2701     foreach my $vmid (keys %$list) {
2702         next if $opt_vmid && ($vmid ne $opt_vmid);
2703         next if !$res->{$vmid}->{pid}; # not running
2704         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2705     }
2706
2707     $qmpclient->queue_execute(undef, 1);
2708
2709     foreach my $vmid (keys %$list) {
2710         next if $opt_vmid && ($vmid ne $opt_vmid);
2711         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2712     }
2713
2714     return $res;
2715 }
2716
2717 sub foreach_drive {
2718     my ($conf, $func, @param) = @_;
2719
2720     foreach my $ds (valid_drive_names()) {
2721         next if !defined($conf->{$ds});
2722
2723         my $drive = parse_drive($ds, $conf->{$ds});
2724         next if !$drive;
2725
2726         &$func($ds, $drive, @param);
2727     }
2728 }
2729
2730 sub foreach_volid {
2731     my ($conf, $func, @param) = @_;
2732
2733     my $volhash = {};
2734
2735     my $test_volid = sub {
2736         my ($volid, $is_cdrom) = @_;
2737
2738         return if !$volid;
2739
2740         $volhash->{$volid} = $is_cdrom || 0;
2741     };
2742
2743     foreach_drive($conf, sub {
2744         my ($ds, $drive) = @_;
2745         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2746     });
2747
2748     foreach my $snapname (keys %{$conf->{snapshots}}) {
2749         my $snap = $conf->{snapshots}->{$snapname};
2750         &$test_volid($snap->{vmstate}, 0);
2751         foreach_drive($snap, sub {
2752             my ($ds, $drive) = @_;
2753             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2754         });
2755     }
2756
2757     foreach my $volid (keys %$volhash) {
2758         &$func($volid, $volhash->{$volid}, @param);
2759     }
2760 }
2761
2762 sub vga_conf_has_spice {
2763     my ($vga) = @_;
2764
2765     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2766
2767     return $1 || 1;
2768 }
2769
2770 sub config_to_command {
2771     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2772
2773     my $cmd = [];
2774     my $globalFlags = [];
2775     my $machineFlags = [];
2776     my $rtcFlags = [];
2777     my $cpuFlags = [];
2778     my $devices = [];
2779     my $pciaddr = '';
2780     my $bridges = {};
2781     my $kvmver = kvm_user_version();
2782     my $vernum = 0; # unknown
2783     my $ostype = $conf->{ostype};
2784     my $winversion = windows_version($ostype);
2785
2786     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2787         $vernum = $1*1000000+$2*1000;
2788     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2789         $vernum = $1*1000000+$2*1000+$3;
2790     }
2791
2792     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2793
2794     my $have_ovz = -f '/proc/vz/vestat';
2795
2796     my $q35 = machine_type_is_q35($conf);
2797     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2798     my $machine_type = $forcemachine || $conf->{machine};
2799     my $use_old_bios_files = undef;
2800     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2801
2802     my $cpuunits = defined($conf->{cpuunits}) ?
2803             $conf->{cpuunits} : $defaults->{cpuunits};
2804
2805     push @$cmd, '/usr/bin/kvm';
2806
2807     push @$cmd, '-id', $vmid;
2808
2809     my $use_virtio = 0;
2810
2811     my $qmpsocket = qmp_socket($vmid);
2812     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2813     push @$cmd, '-mon', "chardev=qmp,mode=control";
2814
2815
2816     push @$cmd, '-pidfile' , pidfile_name($vmid);
2817
2818     push @$cmd, '-daemonize';
2819
2820     if ($conf->{smbios1}) {
2821         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2822     }
2823
2824     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2825         my $ovmfbase;
2826
2827         # prefer the OVMF_CODE variant
2828         if (-f $OVMF_CODE) {
2829             $ovmfbase = $OVMF_CODE;
2830         } elsif (-f $OVMF_IMG) {
2831             $ovmfbase = $OVMF_IMG;
2832         }
2833
2834         die "no uefi base img found\n" if !$ovmfbase;
2835         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmfbase";
2836
2837         if (defined($conf->{efidisk0}) && ($ovmfbase eq $OVMF_CODE)) {
2838             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $conf->{efidisk0});
2839             my $format = $d->{format} // 'raw';
2840             my $path;
2841             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2842             if ($storeid) {
2843                 $path = PVE::Storage::path($storecfg, $d->{file});
2844                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2845                 $format = qemu_img_format($scfg, $volname);
2846             } else {
2847                 $path = $d->{file};
2848                 $format = "raw";
2849             }
2850             push @$cmd, '-drive', "if=pflash,unit=1,id=drive-efidisk0,format=$format,file=$path";
2851         } elsif ($ovmfbase eq $OVMF_CODE) {
2852             warn "using uefi without permanent efivars disk\n";
2853             my $ovmfvar_dst = "/tmp/$vmid-ovmf.fd";
2854             PVE::Tools::file_copy($OVMF_VARS, $ovmfvar_dst, 256*1024);
2855             push @$cmd, '-drive', "if=pflash,unit=1,format=raw,file=$ovmfvar_dst";
2856         } else {
2857             # if the base img is not OVMF_CODE, we do not have to bother
2858             # to create/use a vars image, since it will not be used anyway
2859             # this can only happen if someone manually deletes the OVMF_CODE image
2860             # or has an old pve-qemu-kvm version installed.
2861             # both should not happen, but we ignore it here
2862         }
2863     }
2864
2865
2866     # add usb controllers
2867     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2868     push @$devices, @usbcontrollers if @usbcontrollers;
2869     my $vga = $conf->{vga};
2870
2871     my $qxlnum = vga_conf_has_spice($vga);
2872     $vga = 'qxl' if $qxlnum;
2873
2874     if (!$vga) {
2875         $vga = $winversion >= 6 ? 'std' : 'cirrus';
2876     }
2877
2878     # enable absolute mouse coordinates (needed by vnc)
2879     my $tablet;
2880     if (defined($conf->{tablet})) {
2881         $tablet = $conf->{tablet};
2882     } else {
2883         $tablet = $defaults->{tablet};
2884         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2885         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2886     }
2887
2888     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2889
2890     my $kvm_off = 0;
2891     my $gpu_passthrough;
2892
2893     # host pci devices
2894     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2895         my $d = parse_hostpci($conf->{"hostpci$i"});
2896         next if !$d;
2897
2898         my $pcie = $d->{pcie};
2899         if($pcie){
2900             die "q35 machine model is not enabled" if !$q35;
2901             $pciaddr = print_pcie_addr("hostpci$i");
2902         }else{
2903             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2904         }
2905
2906         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2907         my $xvga = '';
2908         if ($d->{'x-vga'}) {
2909             $xvga = ',x-vga=on';
2910             $kvm_off = 1;
2911             $vga = 'none';
2912             $gpu_passthrough = 1;
2913
2914             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2915                 $xvga = "";
2916             }
2917         }
2918         my $pcidevices = $d->{pciid};
2919         my $multifunction = 1 if @$pcidevices > 1;
2920
2921         my $j=0;
2922         foreach my $pcidevice (@$pcidevices) {
2923
2924             my $id = "hostpci$i";
2925             $id .= ".$j" if $multifunction;
2926             my $addr = $pciaddr;
2927             $addr .= ".$j" if $multifunction;
2928             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2929
2930             if($j == 0){
2931                 $devicestr .= "$rombar$xvga";
2932                 $devicestr .= ",multifunction=on" if $multifunction;
2933             }
2934
2935             push @$devices, '-device', $devicestr;
2936             $j++;
2937         }
2938     }
2939
2940     # usb devices
2941     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
2942     push @$devices, @usbdevices if @usbdevices;
2943     # serial devices
2944     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2945         if (my $path = $conf->{"serial$i"}) {
2946             if ($path eq 'socket') {
2947                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2948                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2949                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2950             } else {
2951                 die "no such serial device\n" if ! -c $path;
2952                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2953                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2954             }
2955         }
2956     }
2957
2958     # parallel devices
2959     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2960         if (my $path = $conf->{"parallel$i"}) {
2961             die "no such parallel device\n" if ! -c $path;
2962             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2963             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2964             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2965         }
2966     }
2967
2968     my $vmname = $conf->{name} || "vm$vmid";
2969
2970     push @$cmd, '-name', $vmname;
2971
2972     my $sockets = 1;
2973     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2974     $sockets = $conf->{sockets} if  $conf->{sockets};
2975
2976     my $cores = $conf->{cores} || 1;
2977
2978     my $maxcpus = $sockets * $cores;
2979
2980     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2981
2982     my $allowed_vcpus = $cpuinfo->{cpus};
2983
2984     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2985         if ($allowed_vcpus < $maxcpus);
2986
2987     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
2988
2989         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2990         for (my $i = 2; $i <= $vcpus; $i++)  {
2991             my $cpustr = print_cpu_device($conf,$i);
2992             push @$cmd, '-device', $cpustr;
2993         }
2994
2995     } else {
2996
2997         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2998     }
2999     push @$cmd, '-nodefaults';
3000
3001     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3002
3003     my $bootindex_hash = {};
3004     my $i = 1;
3005     foreach my $o (split(//, $bootorder)) {
3006         $bootindex_hash->{$o} = $i*100;
3007         $i++;
3008     }
3009
3010     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3011
3012     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3013
3014     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3015
3016     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3017
3018     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3019         my $socket = vnc_socket($vmid);
3020         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3021     } else {
3022         push @$cmd, '-nographic';
3023     }
3024
3025     # time drift fix
3026     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3027
3028     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3029     my $useLocaltime = $conf->{localtime};
3030
3031     if ($winversion >= 5) { # windows
3032         $useLocaltime = 1 if !defined($conf->{localtime});
3033
3034         # use time drift fix when acpi is enabled
3035         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3036             $tdf = 1 if !defined($conf->{tdf});
3037         }
3038     }
3039
3040     if ($winversion >= 6) {
3041         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3042         push @$cmd, '-no-hpet';
3043     }
3044
3045     push @$rtcFlags, 'driftfix=slew' if $tdf;
3046
3047     if ($nokvm) {
3048         push @$machineFlags, 'accel=tcg';
3049     } else {
3050         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3051     }
3052
3053     if ($machine_type) {
3054         push @$machineFlags, "type=${machine_type}";
3055     }
3056
3057     if ($conf->{startdate}) {
3058         push @$rtcFlags, "base=$conf->{startdate}";
3059     } elsif ($useLocaltime) {
3060         push @$rtcFlags, 'base=localtime';
3061     }
3062
3063     my $cpu = $nokvm ? "qemu64" : "kvm64";
3064     if (my $cputype = $conf->{cpu}) {
3065         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3066             or die "Cannot parse cpu description: $cputype\n";
3067         $cpu = $cpuconf->{cputype};
3068         $kvm_off = 1 if $cpuconf->{hidden};
3069     }
3070
3071     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3072
3073     push @$cpuFlags , '-x2apic'
3074         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3075
3076     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3077
3078     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3079
3080     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3081
3082         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3083         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3084     }
3085
3086     add_hyperv_enlighments($cpuFlags, $winversion, $machine_type, $kvmver, $nokvm, $conf->{bios}, $gpu_passthrough);
3087
3088     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3089
3090     push @$cpuFlags, 'kvm=off' if $kvm_off;
3091
3092     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3093         die "internal error"; # should not happen
3094
3095     push @$cpuFlags, "vendor=${cpu_vendor}"
3096         if $cpu_vendor ne 'default';
3097
3098     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3099
3100     push @$cmd, '-cpu', $cpu;
3101
3102     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3103
3104     push @$cmd, '-S' if $conf->{freeze};
3105
3106     # set keyboard layout
3107     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3108     push @$cmd, '-k', $kb if $kb;
3109
3110     # enable sound
3111     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3112     #push @$cmd, '-soundhw', 'es1370';
3113     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3114
3115     if($conf->{agent}) {
3116         my $qgasocket = qmp_socket($vmid, 1);
3117         my $pciaddr = print_pci_addr("qga0", $bridges);
3118         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3119         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3120         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3121     }
3122
3123     my $spice_port;
3124
3125     if ($qxlnum) {
3126         if ($qxlnum > 1) {
3127             if ($winversion){
3128                 for(my $i = 1; $i < $qxlnum; $i++){
3129                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3130                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3131                 }
3132             } else {
3133                 # assume other OS works like Linux
3134                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3135                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3136             }
3137         }
3138
3139         my $pciaddr = print_pci_addr("spice", $bridges);
3140
3141         my $nodename = PVE::INotify::nodename();
3142         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3143         $spice_port = PVE::Tools::next_spice_port($pfamily);
3144
3145         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3146
3147         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3148         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3149         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3150     }
3151
3152     # enable balloon by default, unless explicitly disabled
3153     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3154         $pciaddr = print_pci_addr("balloon0", $bridges);
3155         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3156     }
3157
3158     if ($conf->{watchdog}) {
3159         my $wdopts = parse_watchdog($conf->{watchdog});
3160         $pciaddr = print_pci_addr("watchdog", $bridges);
3161         my $watchdog = $wdopts->{model} || 'i6300esb';
3162         push @$devices, '-device', "$watchdog$pciaddr";
3163         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3164     }
3165
3166     my $vollist = [];
3167     my $scsicontroller = {};
3168     my $ahcicontroller = {};
3169     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3170
3171     # Add iscsi initiator name if available
3172     if (my $initiator = get_initiator_name()) {
3173         push @$devices, '-iscsi', "initiator-name=$initiator";
3174     }
3175
3176     foreach_drive($conf, sub {
3177         my ($ds, $drive) = @_;
3178
3179         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3180             push @$vollist, $drive->{file};
3181         }
3182
3183         $use_virtio = 1 if $ds =~ m/^virtio/;
3184
3185         if (drive_is_cdrom ($drive)) {
3186             if ($bootindex_hash->{d}) {
3187                 $drive->{bootindex} = $bootindex_hash->{d};
3188                 $bootindex_hash->{d} += 1;
3189             }
3190         } else {
3191             if ($bootindex_hash->{c}) {
3192                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3193                 $bootindex_hash->{c} += 1;
3194             }
3195         }
3196
3197         if($drive->{interface} eq 'virtio'){
3198            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3199         }
3200
3201         if ($drive->{interface} eq 'scsi') {
3202
3203             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3204
3205             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3206             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3207
3208             my $iothread = '';
3209             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3210                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3211                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3212             } elsif ($drive->{iothread}) {
3213                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3214             }
3215
3216             my $queues = '';
3217             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3218                 $queues = ",num_queues=$drive->{queues}";
3219             }
3220
3221             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3222             $scsicontroller->{$controller}=1;
3223         }
3224
3225         if ($drive->{interface} eq 'sata') {
3226            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3227            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3228            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3229            $ahcicontroller->{$controller}=1;
3230         }
3231
3232         if ($drive->{interface} eq 'efidisk') {
3233             # this will be added somewhere else
3234             return;
3235         }
3236
3237         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3238         push @$devices, '-drive',$drive_cmd;
3239         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3240     });
3241
3242     for (my $i = 0; $i < $MAX_NETS; $i++) {
3243          next if !$conf->{"net$i"};
3244          my $d = parse_net($conf->{"net$i"});
3245          next if !$d;
3246
3247          $use_virtio = 1 if $d->{model} eq 'virtio';
3248
3249          if ($bootindex_hash->{n}) {
3250             $d->{bootindex} = $bootindex_hash->{n};
3251             $bootindex_hash->{n} += 1;
3252          }
3253
3254          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3255          push @$devices, '-netdev', $netdevfull;
3256
3257          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3258          push @$devices, '-device', $netdevicefull;
3259     }
3260
3261     if (!$q35) {
3262         # add pci bridges
3263         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3264            $bridges->{1} = 1;
3265            $bridges->{2} = 1;
3266         }
3267
3268         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3269
3270         while (my ($k, $v) = each %$bridges) {
3271             $pciaddr = print_pci_addr("pci.$k");
3272             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3273         }
3274     }
3275
3276     # add custom args
3277     if ($conf->{args}) {
3278         my $aa = PVE::Tools::split_args($conf->{args});
3279         push @$cmd, @$aa;
3280     }
3281
3282     push @$cmd, @$devices;
3283     push @$cmd, '-rtc', join(',', @$rtcFlags)
3284         if scalar(@$rtcFlags);
3285     push @$cmd, '-machine', join(',', @$machineFlags)
3286         if scalar(@$machineFlags);
3287     push @$cmd, '-global', join(',', @$globalFlags)
3288         if scalar(@$globalFlags);
3289
3290     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3291 }
3292
3293 sub vnc_socket {
3294     my ($vmid) = @_;
3295     return "${var_run_tmpdir}/$vmid.vnc";
3296 }
3297
3298 sub spice_port {
3299     my ($vmid) = @_;
3300
3301     my $res = vm_mon_cmd($vmid, 'query-spice');
3302
3303     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3304 }
3305
3306 sub qmp_socket {
3307     my ($vmid, $qga) = @_;
3308     my $sockettype = $qga ? 'qga' : 'qmp';
3309     return "${var_run_tmpdir}/$vmid.$sockettype";
3310 }
3311
3312 sub pidfile_name {
3313     my ($vmid) = @_;
3314     return "${var_run_tmpdir}/$vmid.pid";
3315 }
3316
3317 sub vm_devices_list {
3318     my ($vmid) = @_;
3319
3320     my $res = vm_mon_cmd($vmid, 'query-pci');
3321     my $devices = {};
3322     foreach my $pcibus (@$res) {
3323         foreach my $device (@{$pcibus->{devices}}) {
3324             next if !$device->{'qdev_id'};
3325             if ($device->{'pci_bridge'}) {
3326                 $devices->{$device->{'qdev_id'}} = 1;
3327                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3328                     next if !$bridge_device->{'qdev_id'};
3329                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3330                     $devices->{$device->{'qdev_id'}}++;
3331                 }
3332             } else {
3333                 $devices->{$device->{'qdev_id'}} = 1;
3334             }
3335         }
3336     }
3337
3338     my $resblock = vm_mon_cmd($vmid, 'query-block');
3339     foreach my $block (@$resblock) {
3340         if($block->{device} =~ m/^drive-(\S+)/){
3341                 $devices->{$1} = 1;
3342         }
3343     }
3344
3345     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3346     foreach my $mice (@$resmice) {
3347         if ($mice->{name} eq 'QEMU HID Tablet') {
3348             $devices->{tablet} = 1;
3349             last;
3350         }
3351     }
3352
3353     # for usb devices there is no query-usb
3354     # but we can iterate over the entries in
3355     # qom-list path=/machine/peripheral
3356     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3357     foreach my $per (@$resperipheral) {
3358         if ($per->{name} =~ m/^usb\d+$/) {
3359             $devices->{$per->{name}} = 1;
3360         }
3361     }
3362
3363     return $devices;
3364 }
3365
3366 sub vm_deviceplug {
3367     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3368
3369     my $q35 = machine_type_is_q35($conf);
3370
3371     my $devices_list = vm_devices_list($vmid);
3372     return 1 if defined($devices_list->{$deviceid});
3373
3374     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3375
3376     if ($deviceid eq 'tablet') {
3377
3378         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3379
3380     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3381
3382         die "usb hotplug currently not reliable\n";
3383         # since we can't reliably hot unplug all added usb devices
3384         # and usb passthrough disables live migration
3385         # we disable usb hotplugging for now
3386         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3387
3388     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3389
3390         qemu_iothread_add($vmid, $deviceid, $device);
3391
3392         qemu_driveadd($storecfg, $vmid, $device);
3393         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3394
3395         qemu_deviceadd($vmid, $devicefull);
3396         eval { qemu_deviceaddverify($vmid, $deviceid); };
3397         if (my $err = $@) {
3398             eval { qemu_drivedel($vmid, $deviceid); };
3399             warn $@ if $@;
3400             die $err;
3401         }
3402
3403     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3404
3405
3406         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3407         my $pciaddr = print_pci_addr($deviceid);
3408         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3409
3410         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3411
3412         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3413             qemu_iothread_add($vmid, $deviceid, $device);
3414             $devicefull .= ",iothread=iothread-$deviceid";
3415         }
3416
3417         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3418             $devicefull .= ",num_queues=$device->{queues}";
3419         }
3420
3421         qemu_deviceadd($vmid, $devicefull);
3422         qemu_deviceaddverify($vmid, $deviceid);
3423
3424     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3425
3426         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3427         qemu_driveadd($storecfg, $vmid, $device);
3428
3429         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3430         eval { qemu_deviceadd($vmid, $devicefull); };
3431         if (my $err = $@) {
3432             eval { qemu_drivedel($vmid, $deviceid); };
3433             warn $@ if $@;
3434             die $err;
3435         }
3436
3437     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3438
3439         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3440
3441         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3442         my $use_old_bios_files = undef;
3443         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3444
3445         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3446         qemu_deviceadd($vmid, $netdevicefull);
3447         eval { qemu_deviceaddverify($vmid, $deviceid); };
3448         if (my $err = $@) {
3449             eval { qemu_netdevdel($vmid, $deviceid); };
3450             warn $@ if $@;
3451             die $err;
3452         }
3453
3454     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3455
3456         my $bridgeid = $2;
3457         my $pciaddr = print_pci_addr($deviceid);
3458         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3459
3460         qemu_deviceadd($vmid, $devicefull);
3461         qemu_deviceaddverify($vmid, $deviceid);
3462
3463     } else {
3464         die "can't hotplug device '$deviceid'\n";
3465     }
3466
3467     return 1;
3468 }
3469
3470 # fixme: this should raise exceptions on error!
3471 sub vm_deviceunplug {
3472     my ($vmid, $conf, $deviceid) = @_;
3473
3474     my $devices_list = vm_devices_list($vmid);
3475     return 1 if !defined($devices_list->{$deviceid});
3476
3477     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3478
3479     if ($deviceid eq 'tablet') {
3480
3481         qemu_devicedel($vmid, $deviceid);
3482
3483     } elsif ($deviceid =~ m/^usb\d+$/) {
3484
3485         die "usb hotplug currently not reliable\n";
3486         # when unplugging usb devices this way,
3487         # there may be remaining usb controllers/hubs
3488         # so we disable it for now
3489         qemu_devicedel($vmid, $deviceid);
3490         qemu_devicedelverify($vmid, $deviceid);
3491
3492     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3493
3494         qemu_devicedel($vmid, $deviceid);
3495         qemu_devicedelverify($vmid, $deviceid);
3496         qemu_drivedel($vmid, $deviceid);
3497         qemu_iothread_del($conf, $vmid, $deviceid);
3498
3499     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3500
3501         qemu_devicedel($vmid, $deviceid);
3502         qemu_devicedelverify($vmid, $deviceid);
3503         qemu_iothread_del($conf, $vmid, $deviceid);
3504
3505     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3506
3507         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3508         my $device = parse_drive($deviceid, $conf->{$deviceid});
3509         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3510
3511         qemu_devicedel($vmid, $deviceid);
3512         qemu_drivedel($vmid, $deviceid);
3513         qemu_deletescsihw($conf, $vmid, $deviceid);
3514
3515     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3516
3517         qemu_devicedel($vmid, $deviceid);
3518         qemu_devicedelverify($vmid, $deviceid);
3519         qemu_netdevdel($vmid, $deviceid);
3520
3521     } else {
3522         die "can't unplug device '$deviceid'\n";
3523     }
3524
3525     return 1;
3526 }
3527
3528 sub qemu_deviceadd {
3529     my ($vmid, $devicefull) = @_;
3530
3531     $devicefull = "driver=".$devicefull;
3532     my %options =  split(/[=,]/, $devicefull);
3533
3534     vm_mon_cmd($vmid, "device_add" , %options);
3535 }
3536
3537 sub qemu_devicedel {
3538     my ($vmid, $deviceid) = @_;
3539
3540     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3541 }
3542
3543 sub qemu_iothread_add {
3544     my($vmid, $deviceid, $device) = @_;
3545
3546     if ($device->{iothread}) {
3547         my $iothreads = vm_iothreads_list($vmid);
3548         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3549     }
3550 }
3551
3552 sub qemu_iothread_del {
3553     my($conf, $vmid, $deviceid) = @_;
3554
3555     my $device = parse_drive($deviceid, $conf->{$deviceid});
3556     if ($device->{iothread}) {
3557         my $iothreads = vm_iothreads_list($vmid);
3558         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3559     }
3560 }
3561
3562 sub qemu_objectadd {
3563     my($vmid, $objectid, $qomtype) = @_;
3564
3565     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3566
3567     return 1;
3568 }
3569
3570 sub qemu_objectdel {
3571     my($vmid, $objectid) = @_;
3572
3573     vm_mon_cmd($vmid, "object-del", id => $objectid);
3574
3575     return 1;
3576 }
3577
3578 sub qemu_driveadd {
3579     my ($storecfg, $vmid, $device) = @_;
3580
3581     my $drive = print_drive_full($storecfg, $vmid, $device);
3582     $drive =~ s/\\/\\\\/g;
3583     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3584
3585     # If the command succeeds qemu prints: "OK"
3586     return 1 if $ret =~ m/OK/s;
3587
3588     die "adding drive failed: $ret\n";
3589 }
3590
3591 sub qemu_drivedel {
3592     my($vmid, $deviceid) = @_;
3593
3594     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3595     $ret =~ s/^\s+//;
3596
3597     return 1 if $ret eq "";
3598
3599     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3600     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3601
3602     die "deleting drive $deviceid failed : $ret\n";
3603 }
3604
3605 sub qemu_deviceaddverify {
3606     my ($vmid, $deviceid) = @_;
3607
3608     for (my $i = 0; $i <= 5; $i++) {
3609          my $devices_list = vm_devices_list($vmid);
3610          return 1 if defined($devices_list->{$deviceid});
3611          sleep 1;
3612     }
3613
3614     die "error on hotplug device '$deviceid'\n";
3615 }
3616
3617
3618 sub qemu_devicedelverify {
3619     my ($vmid, $deviceid) = @_;
3620
3621     # need to verify that the device is correctly removed as device_del
3622     # is async and empty return is not reliable
3623
3624     for (my $i = 0; $i <= 5; $i++) {
3625          my $devices_list = vm_devices_list($vmid);
3626          return 1 if !defined($devices_list->{$deviceid});
3627          sleep 1;
3628     }
3629
3630     die "error on hot-unplugging device '$deviceid'\n";
3631 }
3632
3633 sub qemu_findorcreatescsihw {
3634     my ($storecfg, $conf, $vmid, $device) = @_;
3635
3636     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3637
3638     my $scsihwid="$controller_prefix$controller";
3639     my $devices_list = vm_devices_list($vmid);
3640
3641     if(!defined($devices_list->{$scsihwid})) {
3642         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3643     }
3644
3645     return 1;
3646 }
3647
3648 sub qemu_deletescsihw {
3649     my ($conf, $vmid, $opt) = @_;
3650
3651     my $device = parse_drive($opt, $conf->{$opt});
3652
3653     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3654         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3655         return 1;
3656     }
3657
3658     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3659
3660     my $devices_list = vm_devices_list($vmid);
3661     foreach my $opt (keys %{$devices_list}) {
3662         if (PVE::QemuServer::is_valid_drivename($opt)) {
3663             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3664             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3665                 return 1;
3666             }
3667         }
3668     }
3669
3670     my $scsihwid="scsihw$controller";
3671
3672     vm_deviceunplug($vmid, $conf, $scsihwid);
3673
3674     return 1;
3675 }
3676
3677 sub qemu_add_pci_bridge {
3678     my ($storecfg, $conf, $vmid, $device) = @_;
3679
3680     my $bridges = {};
3681
3682     my $bridgeid;
3683
3684     print_pci_addr($device, $bridges);
3685
3686     while (my ($k, $v) = each %$bridges) {
3687         $bridgeid = $k;
3688     }
3689     return 1 if !defined($bridgeid) || $bridgeid < 1;
3690
3691     my $bridge = "pci.$bridgeid";
3692     my $devices_list = vm_devices_list($vmid);
3693
3694     if (!defined($devices_list->{$bridge})) {
3695         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3696     }
3697
3698     return 1;
3699 }
3700
3701 sub qemu_set_link_status {
3702     my ($vmid, $device, $up) = @_;
3703
3704     vm_mon_cmd($vmid, "set_link", name => $device,
3705                up => $up ? JSON::true : JSON::false);
3706 }
3707
3708 sub qemu_netdevadd {
3709     my ($vmid, $conf, $device, $deviceid) = @_;
3710
3711     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3712     my %options =  split(/[=,]/, $netdev);
3713
3714     vm_mon_cmd($vmid, "netdev_add",  %options);
3715     return 1;
3716 }
3717
3718 sub qemu_netdevdel {
3719     my ($vmid, $deviceid) = @_;
3720
3721     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3722 }
3723
3724 sub qemu_usb_hotplug {
3725     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3726
3727     return if !$device;
3728
3729     # remove the old one first
3730     vm_deviceunplug($vmid, $conf, $deviceid);
3731
3732     # check if xhci controller is necessary and available
3733     if ($device->{usb3}) {
3734
3735         my $devicelist = vm_devices_list($vmid);
3736
3737         if (!$devicelist->{xhci}) {
3738             my $pciaddr = print_pci_addr("xhci");
3739             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3740         }
3741     }
3742     my $d = parse_usb_device($device->{host});
3743     $d->{usb3} = $device->{usb3};
3744
3745     # add the new one
3746     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3747 }
3748
3749 sub qemu_cpu_hotplug {
3750     my ($vmid, $conf, $vcpus) = @_;
3751
3752     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3753
3754     my $sockets = 1;
3755     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3756     $sockets = $conf->{sockets} if  $conf->{sockets};
3757     my $cores = $conf->{cores} || 1;
3758     my $maxcpus = $sockets * $cores;
3759
3760     $vcpus = $maxcpus if !$vcpus;
3761
3762     die "you can't add more vcpus than maxcpus\n"
3763         if $vcpus > $maxcpus;
3764
3765     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3766
3767     if ($vcpus < $currentvcpus) {
3768
3769         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3770
3771             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3772                 qemu_devicedel($vmid, "cpu$i");
3773                 my $retry = 0;
3774                 my $currentrunningvcpus = undef;
3775                 while (1) {
3776                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3777                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3778                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3779                     $retry++;
3780                     sleep 1;
3781                 }
3782                 #update conf after each succesfull cpu unplug
3783                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3784                 PVE::QemuConfig->write_config($vmid, $conf);
3785             }
3786         } else {
3787             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3788         }
3789
3790         return;
3791     }
3792
3793     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3794     die "vcpus in running vm does not match its configuration\n"
3795         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3796
3797     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3798
3799         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3800             my $cpustr = print_cpu_device($conf, $i);
3801             qemu_deviceadd($vmid, $cpustr);
3802
3803             my $retry = 0;
3804             my $currentrunningvcpus = undef;
3805             while (1) {
3806                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3807                 last if scalar(@{$currentrunningvcpus}) == $i;
3808                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3809                 sleep 1;
3810                 $retry++;
3811             }
3812             #update conf after each succesfull cpu hotplug
3813             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3814             PVE::QemuConfig->write_config($vmid, $conf);
3815         }
3816     } else {
3817
3818         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3819             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3820         }
3821     }
3822 }
3823
3824 sub qemu_block_set_io_throttle {
3825     my ($vmid, $deviceid,
3826         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3827         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3828         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3829         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3830
3831     return if !check_running($vmid) ;
3832
3833     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3834         bps => int($bps),
3835         bps_rd => int($bps_rd),
3836         bps_wr => int($bps_wr),
3837         iops => int($iops),
3838         iops_rd => int($iops_rd),
3839         iops_wr => int($iops_wr),
3840         bps_max => int($bps_max),
3841         bps_rd_max => int($bps_rd_max),
3842         bps_wr_max => int($bps_wr_max),
3843         iops_max => int($iops_max),
3844         iops_rd_max => int($iops_rd_max),
3845         iops_wr_max => int($iops_wr_max),
3846         bps_max_length => int($bps_max_length),
3847         bps_rd_max_length => int($bps_rd_max_length),
3848         bps_wr_max_length => int($bps_wr_max_length),
3849         iops_max_length => int($iops_max_length),
3850         iops_rd_max_length => int($iops_rd_max_length),
3851         iops_wr_max_length => int($iops_wr_max_length),
3852     );
3853
3854 }
3855
3856 # old code, only used to shutdown old VM after update
3857 sub __read_avail {
3858     my ($fh, $timeout) = @_;
3859
3860     my $sel = new IO::Select;
3861     $sel->add($fh);
3862
3863     my $res = '';
3864     my $buf;
3865
3866     my @ready;
3867     while (scalar (@ready = $sel->can_read($timeout))) {
3868         my $count;
3869         if ($count = $fh->sysread($buf, 8192)) {
3870             if ($buf =~ /^(.*)\(qemu\) $/s) {
3871                 $res .= $1;
3872                 last;
3873             } else {
3874                 $res .= $buf;
3875             }
3876         } else {
3877             if (!defined($count)) {
3878                 die "$!\n";
3879             }
3880             last;
3881         }
3882     }
3883
3884     die "monitor read timeout\n" if !scalar(@ready);
3885
3886     return $res;
3887 }
3888
3889 # old code, only used to shutdown old VM after update
3890 sub vm_monitor_command {
3891     my ($vmid, $cmdstr, $nocheck) = @_;
3892
3893     my $res;
3894
3895     eval {
3896         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3897
3898         my $sname = "${var_run_tmpdir}/$vmid.mon";
3899
3900         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3901             die "unable to connect to VM $vmid socket - $!\n";
3902
3903         my $timeout = 3;
3904
3905         # hack: migrate sometime blocks the monitor (when migrate_downtime
3906         # is set)
3907         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3908             $timeout = 60*60; # 1 hour
3909         }
3910
3911         # read banner;
3912         my $data = __read_avail($sock, $timeout);
3913
3914         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3915             die "got unexpected qemu monitor banner\n";
3916         }
3917
3918         my $sel = new IO::Select;
3919         $sel->add($sock);
3920
3921         if (!scalar(my @ready = $sel->can_write($timeout))) {
3922             die "monitor write error - timeout";
3923         }
3924
3925         my $fullcmd = "$cmdstr\r";
3926
3927         # syslog('info', "VM $vmid monitor command: $cmdstr");
3928
3929         my $b;
3930         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3931             die "monitor write error - $!";
3932         }
3933
3934         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3935
3936         $timeout = 20;
3937
3938         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3939             $timeout = 60*60; # 1 hour
3940         } elsif ($cmdstr =~ m/^(eject|change)/) {
3941             $timeout = 60; # note: cdrom mount command is slow
3942         }
3943         if ($res = __read_avail($sock, $timeout)) {
3944
3945             my @lines = split("\r?\n", $res);
3946
3947             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3948
3949             $res = join("\n", @lines);
3950             $res .= "\n";
3951         }
3952     };
3953
3954     my $err = $@;
3955
3956     if ($err) {
3957         syslog("err", "VM $vmid monitor command failed - $err");
3958         die $err;
3959     }
3960
3961     return $res;
3962 }
3963
3964 sub qemu_block_resize {
3965     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3966
3967     my $running = check_running($vmid);
3968
3969     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3970
3971     return if !$running;
3972
3973     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3974
3975 }
3976
3977 sub qemu_volume_snapshot {
3978     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3979
3980     my $running = check_running($vmid);
3981
3982     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3983         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3984     } else {
3985         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3986     }
3987 }
3988
3989 sub qemu_volume_snapshot_delete {
3990     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3991
3992     my $running = check_running($vmid);
3993
3994     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3995         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3996     } else {
3997         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3998     }
3999 }
4000
4001 sub set_migration_caps {
4002     my ($vmid) = @_;
4003
4004     my $cap_ref = [];
4005
4006     my $enabled_cap = {
4007         "auto-converge" => 1,
4008         "xbzrle" => 1,
4009         "x-rdma-pin-all" => 0,
4010         "zero-blocks" => 0,
4011         "compress" => 0
4012     };
4013
4014     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4015
4016     for my $supported_capability (@$supported_capabilities) {
4017         push @$cap_ref, {
4018             capability => $supported_capability->{capability},
4019             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4020         };
4021     }
4022
4023     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4024 }
4025
4026 my $fast_plug_option = {
4027     'lock' => 1,
4028     'name' => 1,
4029     'onboot' => 1,
4030     'shares' => 1,
4031     'startup' => 1,
4032     'description' => 1,
4033     'protection' => 1,
4034 };
4035
4036 # hotplug changes in [PENDING]
4037 # $selection hash can be used to only apply specified options, for
4038 # example: { cores => 1 } (only apply changed 'cores')
4039 # $errors ref is used to return error messages
4040 sub vmconfig_hotplug_pending {
4041     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4042
4043     my $defaults = load_defaults();
4044
4045     # commit values which do not have any impact on running VM first
4046     # Note: those option cannot raise errors, we we do not care about
4047     # $selection and always apply them.
4048
4049     my $add_error = sub {
4050         my ($opt, $msg) = @_;
4051         $errors->{$opt} = "hotplug problem - $msg";
4052     };
4053
4054     my $changes = 0;
4055     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4056         if ($fast_plug_option->{$opt}) {
4057             $conf->{$opt} = $conf->{pending}->{$opt};
4058             delete $conf->{pending}->{$opt};
4059             $changes = 1;
4060         }
4061     }
4062
4063     if ($changes) {
4064         PVE::QemuConfig->write_config($vmid, $conf);
4065         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4066     }
4067
4068     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4069
4070     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4071     while (my ($opt, $force) = each %$pending_delete_hash) {
4072         next if $selection && !$selection->{$opt};
4073         eval {
4074             if ($opt eq 'hotplug') {
4075                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4076             } elsif ($opt eq 'tablet') {
4077                 die "skip\n" if !$hotplug_features->{usb};
4078                 if ($defaults->{tablet}) {
4079                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4080                 } else {
4081                     vm_deviceunplug($vmid, $conf, $opt);
4082                 }
4083             } elsif ($opt =~ m/^usb\d+/) {
4084                 die "skip\n";
4085                 # since we cannot reliably hot unplug usb devices
4086                 # we are disabling it
4087                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4088                 vm_deviceunplug($vmid, $conf, $opt);
4089             } elsif ($opt eq 'vcpus') {
4090                 die "skip\n" if !$hotplug_features->{cpu};
4091                 qemu_cpu_hotplug($vmid, $conf, undef);
4092             } elsif ($opt eq 'balloon') {
4093                 # enable balloon device is not hotpluggable
4094                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4095             } elsif ($fast_plug_option->{$opt}) {
4096                 # do nothing
4097             } elsif ($opt =~ m/^net(\d+)$/) {
4098                 die "skip\n" if !$hotplug_features->{network};
4099                 vm_deviceunplug($vmid, $conf, $opt);
4100             } elsif (is_valid_drivename($opt)) {
4101                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4102                 vm_deviceunplug($vmid, $conf, $opt);
4103                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4104             } elsif ($opt =~ m/^memory$/) {
4105                 die "skip\n" if !$hotplug_features->{memory};
4106                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4107             } elsif ($opt eq 'cpuunits') {
4108                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4109             } elsif ($opt eq 'cpulimit') {
4110                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4111             } else {
4112                 die "skip\n";
4113             }
4114         };
4115         if (my $err = $@) {
4116             &$add_error($opt, $err) if $err ne "skip\n";
4117         } else {
4118             # save new config if hotplug was successful
4119             delete $conf->{$opt};
4120             vmconfig_undelete_pending_option($conf, $opt);
4121             PVE::QemuConfig->write_config($vmid, $conf);
4122             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4123         }
4124     }
4125
4126     foreach my $opt (keys %{$conf->{pending}}) {
4127         next if $selection && !$selection->{$opt};
4128         my $value = $conf->{pending}->{$opt};
4129         eval {
4130             if ($opt eq 'hotplug') {
4131                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4132             } elsif ($opt eq 'tablet') {
4133                 die "skip\n" if !$hotplug_features->{usb};
4134                 if ($value == 1) {
4135                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4136                 } elsif ($value == 0) {
4137                     vm_deviceunplug($vmid, $conf, $opt);
4138                 }
4139             } elsif ($opt =~ m/^usb\d+$/) {
4140                 die "skip\n";
4141                 # since we cannot reliably hot unplug usb devices
4142                 # we are disabling it
4143                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4144                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4145                 die "skip\n" if !$d;
4146                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4147             } elsif ($opt eq 'vcpus') {
4148                 die "skip\n" if !$hotplug_features->{cpu};
4149                 qemu_cpu_hotplug($vmid, $conf, $value);
4150             } elsif ($opt eq 'balloon') {
4151                 # enable/disable balloning device is not hotpluggable
4152                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4153                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4154                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4155
4156                 # allow manual ballooning if shares is set to zero
4157                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4158                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4159                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4160                 }
4161             } elsif ($opt =~ m/^net(\d+)$/) {
4162                 # some changes can be done without hotplug
4163                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4164                                     $vmid, $opt, $value);
4165             } elsif (is_valid_drivename($opt)) {
4166                 # some changes can be done without hotplug
4167                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4168                                      $vmid, $opt, $value, 1);
4169             } elsif ($opt =~ m/^memory$/) { #dimms
4170                 die "skip\n" if !$hotplug_features->{memory};
4171                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4172             } elsif ($opt eq 'cpuunits') {
4173                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4174             } elsif ($opt eq 'cpulimit') {
4175                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4176                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4177             } else {
4178                 die "skip\n";  # skip non-hot-pluggable options
4179             }
4180         };
4181         if (my $err = $@) {
4182             &$add_error($opt, $err) if $err ne "skip\n";
4183         } else {
4184             # save new config if hotplug was successful
4185             $conf->{$opt} = $value;
4186             delete $conf->{pending}->{$opt};
4187             PVE::QemuConfig->write_config($vmid, $conf);
4188             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4189         }
4190     }
4191 }
4192
4193 sub try_deallocate_drive {
4194     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4195
4196     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4197         my $volid = $drive->{file};
4198         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4199             my $sid = PVE::Storage::parse_volume_id($volid);
4200             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4201
4202             # check if the disk is really unused
4203             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4204                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4205             PVE::Storage::vdisk_free($storecfg, $volid);
4206             return 1;
4207         } else {
4208             # If vm is not owner of this disk remove from config
4209             return 1;
4210         }
4211     }
4212
4213     return undef;
4214 }
4215
4216 sub vmconfig_delete_or_detach_drive {
4217     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4218
4219     my $drive = parse_drive($opt, $conf->{$opt});
4220
4221     my $rpcenv = PVE::RPCEnvironment::get();
4222     my $authuser = $rpcenv->get_user();
4223
4224     if ($force) {
4225         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4226         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4227     } else {
4228         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4229     }
4230 }
4231
4232 sub vmconfig_apply_pending {
4233     my ($vmid, $conf, $storecfg) = @_;
4234
4235     # cold plug
4236
4237     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4238     while (my ($opt, $force) = each %$pending_delete_hash) {
4239         die "internal error" if $opt =~ m/^unused/;
4240         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4241         if (!defined($conf->{$opt})) {
4242             vmconfig_undelete_pending_option($conf, $opt);
4243             PVE::QemuConfig->write_config($vmid, $conf);
4244         } elsif (is_valid_drivename($opt)) {
4245             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4246             vmconfig_undelete_pending_option($conf, $opt);
4247             delete $conf->{$opt};
4248             PVE::QemuConfig->write_config($vmid, $conf);
4249         } else {
4250             vmconfig_undelete_pending_option($conf, $opt);
4251             delete $conf->{$opt};
4252             PVE::QemuConfig->write_config($vmid, $conf);
4253         }
4254     }
4255
4256     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4257
4258     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4259         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4260
4261         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4262             # skip if nothing changed
4263         } elsif (is_valid_drivename($opt)) {
4264             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4265                 if defined($conf->{$opt});
4266             $conf->{$opt} = $conf->{pending}->{$opt};
4267         } else {
4268             $conf->{$opt} = $conf->{pending}->{$opt};
4269         }
4270
4271         delete $conf->{pending}->{$opt};
4272         PVE::QemuConfig->write_config($vmid, $conf);
4273     }
4274 }
4275
4276 my $safe_num_ne = sub {
4277     my ($a, $b) = @_;
4278
4279     return 0 if !defined($a) && !defined($b);
4280     return 1 if !defined($a);
4281     return 1 if !defined($b);
4282
4283     return $a != $b;
4284 };
4285
4286 my $safe_string_ne = sub {
4287     my ($a, $b) = @_;
4288
4289     return 0 if !defined($a) && !defined($b);
4290     return 1 if !defined($a);
4291     return 1 if !defined($b);
4292
4293     return $a ne $b;
4294 };
4295
4296 sub vmconfig_update_net {
4297     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4298
4299     my $newnet = parse_net($value);
4300
4301     if ($conf->{$opt}) {
4302         my $oldnet = parse_net($conf->{$opt});
4303
4304         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4305             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4306             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4307             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4308
4309             # for non online change, we try to hot-unplug
4310             die "skip\n" if !$hotplug;
4311             vm_deviceunplug($vmid, $conf, $opt);
4312         } else {
4313
4314             die "internal error" if $opt !~ m/net(\d+)/;
4315             my $iface = "tap${vmid}i$1";
4316
4317             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4318                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4319                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4320                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4321                 PVE::Network::tap_unplug($iface);
4322                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4323             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4324                 # Rate can be applied on its own but any change above needs to
4325                 # include the rate in tap_plug since OVS resets everything.
4326                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4327             }
4328
4329             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4330                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4331             }
4332
4333             return 1;
4334         }
4335     }
4336
4337     if ($hotplug) {
4338         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4339     } else {
4340         die "skip\n";
4341     }
4342 }
4343
4344 sub vmconfig_update_disk {
4345     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4346
4347     # fixme: do we need force?
4348
4349     my $drive = parse_drive($opt, $value);
4350
4351     if ($conf->{$opt}) {
4352
4353         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4354
4355             my $media = $drive->{media} || 'disk';
4356             my $oldmedia = $old_drive->{media} || 'disk';
4357             die "unable to change media type\n" if $media ne $oldmedia;
4358
4359             if (!drive_is_cdrom($old_drive)) {
4360
4361                 if ($drive->{file} ne $old_drive->{file}) {
4362
4363                     die "skip\n" if !$hotplug;
4364
4365                     # unplug and register as unused
4366                     vm_deviceunplug($vmid, $conf, $opt);
4367                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4368
4369                 } else {
4370                     # update existing disk
4371
4372                     # skip non hotpluggable value
4373                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4374                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4375                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4376                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4377                         die "skip\n";
4378                     }
4379
4380                     # apply throttle
4381                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4382                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4383                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4384                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4385                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4386                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4387                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4388                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4389                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4390                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4391                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4392                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4393                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4394                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4395                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4396                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4397                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4398                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4399
4400                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4401                                                    ($drive->{mbps} || 0)*1024*1024,
4402                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4403                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4404                                                    $drive->{iops} || 0,
4405                                                    $drive->{iops_rd} || 0,
4406                                                    $drive->{iops_wr} || 0,
4407                                                    ($drive->{mbps_max} || 0)*1024*1024,
4408                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4409                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4410                                                    $drive->{iops_max} || 0,
4411                                                    $drive->{iops_rd_max} || 0,
4412                                                    $drive->{iops_wr_max} || 0,
4413                                                    $drive->{bps_max_length} || 1,
4414                                                    $drive->{bps_rd_max_length} || 1,
4415                                                    $drive->{bps_wr_max_length} || 1,
4416                                                    $drive->{iops_max_length} || 1,
4417                                                    $drive->{iops_rd_max_length} || 1,
4418                                                    $drive->{iops_wr_max_length} || 1);
4419
4420                     }
4421
4422                     return 1;
4423                 }
4424
4425             } else { # cdrom
4426
4427                 if ($drive->{file} eq 'none') {
4428                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4429                 } else {
4430                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4431                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4432                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4433                 }
4434
4435                 return 1;
4436             }
4437         }
4438     }
4439
4440     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4441     # hotplug new disks
4442     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4443     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4444 }
4445
4446 sub vm_start {
4447     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4448         $forcemachine, $spice_ticket, $migration_network, $migration_type) = @_;
4449
4450     PVE::QemuConfig->lock_config($vmid, sub {
4451         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4452
4453         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4454
4455         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4456
4457         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4458
4459         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4460             vmconfig_apply_pending($vmid, $conf, $storecfg);
4461             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4462         }
4463
4464         my $defaults = load_defaults();
4465
4466         # set environment variable useful inside network script
4467         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4468
4469         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4470
4471         my $migrate_port = 0;
4472         my $migrate_uri;
4473         if ($statefile) {
4474             if ($statefile eq 'tcp') {
4475                 my $localip = "localhost";
4476                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4477                 my $nodename = PVE::INotify::nodename();
4478
4479                 if (!defined($migration_type)) {
4480                     if (defined($datacenterconf->{migration}->{type})) {
4481                         $migration_type = $datacenterconf->{migration}->{type};
4482                     } else {
4483                         $migration_type = 'secure';
4484                     }
4485                 }
4486
4487                 if ($migration_type eq 'insecure') {
4488                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4489                     if ($migrate_network_addr) {
4490                         $localip = $migrate_network_addr;
4491                     } else {
4492                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4493                     }
4494
4495                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4496                 }
4497
4498                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4499                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4500                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4501                 push @$cmd, '-incoming', $migrate_uri;
4502                 push @$cmd, '-S';
4503
4504             } elsif ($statefile eq 'unix') {
4505                 # should be default for secure migrations as a ssh TCP forward
4506                 # tunnel is not deterministic reliable ready and fails regurarly
4507                 # to set up in time, so use UNIX socket forwards
4508                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4509                 unlink $socket_addr;
4510
4511                 $migrate_uri = "unix:$socket_addr";
4512
4513                 push @$cmd, '-incoming', $migrate_uri;
4514                 push @$cmd, '-S';
4515
4516             } else {
4517                 push @$cmd, '-loadstate', $statefile;
4518             }
4519         } elsif ($paused) {
4520             push @$cmd, '-S';
4521         }
4522
4523         # host pci devices
4524         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4525           my $d = parse_hostpci($conf->{"hostpci$i"});
4526           next if !$d;
4527           my $pcidevices = $d->{pciid};
4528           foreach my $pcidevice (@$pcidevices) {
4529                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4530
4531                 my $info = pci_device_info("0000:$pciid");
4532                 die "IOMMU not present\n" if !check_iommu_support();
4533                 die "no pci device info for device '$pciid'\n" if !$info;
4534                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4535                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4536           }
4537         }
4538
4539         PVE::Storage::activate_volumes($storecfg, $vollist);
4540
4541         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4542             my $cmd = [];
4543             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4544             eval  { run_command($cmd); };
4545         }
4546
4547         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4548                                                   : $defaults->{cpuunits};
4549
4550         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4551
4552         my %properties = (
4553             Slice => 'qemu.slice',
4554             KillMode => 'none',
4555             CPUShares => $cpuunits
4556         );
4557
4558         if (my $cpulimit = $conf->{cpulimit}) {
4559             $properties{CPUQuota} = int($cpulimit * 100);
4560         }
4561         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4562
4563         if ($conf->{hugepages}) {
4564
4565             my $code = sub {
4566                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4567                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4568
4569                 PVE::QemuServer::Memory::hugepages_mount();
4570                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4571
4572                 eval  {
4573                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4574                     run_command($cmd, %run_params);
4575                 };
4576
4577                 if (my $err = $@) {
4578                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4579                     die $err;
4580                 }
4581
4582                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4583             };
4584             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4585
4586         } else {
4587             eval  {
4588                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4589                 run_command($cmd, %run_params);
4590             };
4591         }
4592
4593         if (my $err = $@) {
4594             # deactivate volumes if start fails
4595             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4596             die "start failed: $err";
4597         }
4598
4599         print "migration listens on $migrate_uri\n" if $migrate_uri;
4600
4601         if ($statefile && $statefile ne 'tcp')  {
4602             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4603             warn $@ if $@;
4604         }
4605
4606         if ($migratedfrom) {
4607
4608             eval {
4609                 set_migration_caps($vmid);
4610             };
4611             warn $@ if $@;
4612
4613             if ($spice_port) {
4614                 print "spice listens on port $spice_port\n";
4615                 if ($spice_ticket) {
4616                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4617                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4618                 }
4619             }
4620
4621         } else {
4622
4623             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4624                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4625                     if $conf->{balloon};
4626             }
4627
4628             foreach my $opt (keys %$conf) {
4629                 next if $opt !~  m/^net\d+$/;
4630                 my $nicconf = parse_net($conf->{$opt});
4631                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4632             }
4633         }
4634
4635         vm_mon_cmd_nocheck($vmid, 'qom-set',
4636                     path => "machine/peripheral/balloon0",
4637                     property => "guest-stats-polling-interval",
4638                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4639
4640     });
4641 }
4642
4643 sub vm_mon_cmd {
4644     my ($vmid, $execute, %params) = @_;
4645
4646     my $cmd = { execute => $execute, arguments => \%params };
4647     vm_qmp_command($vmid, $cmd);
4648 }
4649
4650 sub vm_mon_cmd_nocheck {
4651     my ($vmid, $execute, %params) = @_;
4652
4653     my $cmd = { execute => $execute, arguments => \%params };
4654     vm_qmp_command($vmid, $cmd, 1);
4655 }
4656
4657 sub vm_qmp_command {
4658     my ($vmid, $cmd, $nocheck) = @_;
4659
4660     my $res;
4661
4662     my $timeout;
4663     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4664         $timeout = $cmd->{arguments}->{timeout};
4665         delete $cmd->{arguments}->{timeout};
4666     }
4667
4668     eval {
4669         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4670         my $sname = qmp_socket($vmid);
4671         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4672             my $qmpclient = PVE::QMPClient->new();
4673
4674             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4675         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4676             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4677                 if scalar(%{$cmd->{arguments}});
4678             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4679         } else {
4680             die "unable to open monitor socket\n";
4681         }
4682     };
4683     if (my $err = $@) {
4684         syslog("err", "VM $vmid qmp command failed - $err");
4685         die $err;
4686     }
4687
4688     return $res;
4689 }
4690
4691 sub vm_human_monitor_command {
4692     my ($vmid, $cmdline) = @_;
4693
4694     my $res;
4695
4696     my $cmd = {
4697         execute => 'human-monitor-command',
4698         arguments => { 'command-line' => $cmdline},
4699     };
4700
4701     return vm_qmp_command($vmid, $cmd);
4702 }
4703
4704 sub vm_commandline {
4705     my ($storecfg, $vmid) = @_;
4706
4707     my $conf = PVE::QemuConfig->load_config($vmid);
4708
4709     my $defaults = load_defaults();
4710
4711     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4712
4713     return PVE::Tools::cmd2string($cmd);
4714 }
4715
4716 sub vm_reset {
4717     my ($vmid, $skiplock) = @_;
4718
4719     PVE::QemuConfig->lock_config($vmid, sub {
4720
4721         my $conf = PVE::QemuConfig->load_config($vmid);
4722
4723         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4724
4725         vm_mon_cmd($vmid, "system_reset");
4726     });
4727 }
4728
4729 sub get_vm_volumes {
4730     my ($conf) = @_;
4731
4732     my $vollist = [];
4733     foreach_volid($conf, sub {
4734         my ($volid, $is_cdrom) = @_;
4735
4736         return if $volid =~ m|^/|;
4737
4738         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4739         return if !$sid;
4740
4741         push @$vollist, $volid;
4742     });
4743
4744     return $vollist;
4745 }
4746
4747 sub vm_stop_cleanup {
4748     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4749
4750     eval {
4751
4752         if (!$keepActive) {
4753             my $vollist = get_vm_volumes($conf);
4754             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4755         }
4756
4757         foreach my $ext (qw(mon qmp pid vnc qga)) {
4758             unlink "/var/run/qemu-server/${vmid}.$ext";
4759         }
4760
4761         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4762     };
4763     warn $@ if $@; # avoid errors - just warn
4764 }
4765
4766 # Note: use $nockeck to skip tests if VM configuration file exists.
4767 # We need that when migration VMs to other nodes (files already moved)
4768 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4769 sub vm_stop {
4770     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4771
4772     $force = 1 if !defined($force) && !$shutdown;
4773
4774     if ($migratedfrom){
4775         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4776         kill 15, $pid if $pid;
4777         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4778         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4779         return;
4780     }
4781
4782     PVE::QemuConfig->lock_config($vmid, sub {
4783
4784         my $pid = check_running($vmid, $nocheck);
4785         return if !$pid;
4786
4787         my $conf;
4788         if (!$nocheck) {
4789             $conf = PVE::QemuConfig->load_config($vmid);
4790             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4791             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4792                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4793                 $timeout = $opts->{down} if $opts->{down};
4794             }
4795         }
4796
4797         $timeout = 60 if !defined($timeout);
4798
4799         eval {
4800             if ($shutdown) {
4801                 if (defined($conf) && $conf->{agent}) {
4802                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4803                 } else {
4804                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4805                 }
4806             } else {
4807                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4808             }
4809         };
4810         my $err = $@;
4811
4812         if (!$err) {
4813             my $count = 0;
4814             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4815                 $count++;
4816                 sleep 1;
4817             }
4818
4819             if ($count >= $timeout) {
4820                 if ($force) {
4821                     warn "VM still running - terminating now with SIGTERM\n";
4822                     kill 15, $pid;
4823                 } else {
4824                     die "VM quit/powerdown failed - got timeout\n";
4825                 }
4826             } else {
4827                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4828                 return;
4829             }
4830         } else {
4831             if ($force) {
4832                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4833                 kill 15, $pid;
4834             } else {
4835                 die "VM quit/powerdown failed\n";
4836             }
4837         }
4838
4839         # wait again
4840         $timeout = 10;
4841
4842         my $count = 0;
4843         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4844             $count++;
4845             sleep 1;
4846         }
4847
4848         if ($count >= $timeout) {
4849             warn "VM still running - terminating now with SIGKILL\n";
4850             kill 9, $pid;
4851             sleep 1;
4852         }
4853
4854         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4855    });
4856 }
4857
4858 sub vm_suspend {
4859     my ($vmid, $skiplock) = @_;
4860
4861     PVE::QemuConfig->lock_config($vmid, sub {
4862
4863         my $conf = PVE::QemuConfig->load_config($vmid);
4864
4865         PVE::QemuConfig->check_lock($conf)
4866             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4867
4868         vm_mon_cmd($vmid, "stop");
4869     });
4870 }
4871
4872 sub vm_resume {
4873     my ($vmid, $skiplock, $nocheck) = @_;
4874
4875     PVE::QemuConfig->lock_config($vmid, sub {
4876
4877         if (!$nocheck) {
4878
4879             my $conf = PVE::QemuConfig->load_config($vmid);
4880
4881             PVE::QemuConfig->check_lock($conf)
4882                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4883
4884             vm_mon_cmd($vmid, "cont");
4885
4886         } else {
4887             vm_mon_cmd_nocheck($vmid, "cont");
4888         }
4889     });
4890 }
4891
4892 sub vm_sendkey {
4893     my ($vmid, $skiplock, $key) = @_;
4894
4895     PVE::QemuConfig->lock_config($vmid, sub {
4896
4897         my $conf = PVE::QemuConfig->load_config($vmid);
4898
4899         # there is no qmp command, so we use the human monitor command
4900         vm_human_monitor_command($vmid, "sendkey $key");
4901     });
4902 }
4903
4904 sub vm_destroy {
4905     my ($storecfg, $vmid, $skiplock) = @_;
4906
4907     PVE::QemuConfig->lock_config($vmid, sub {
4908
4909         my $conf = PVE::QemuConfig->load_config($vmid);
4910
4911         if (!check_running($vmid)) {
4912             destroy_vm($storecfg, $vmid, undef, $skiplock);
4913         } else {
4914             die "VM $vmid is running - destroy failed\n";
4915         }
4916     });
4917 }
4918
4919 # pci helpers
4920
4921 sub file_write {
4922     my ($filename, $buf) = @_;
4923
4924     my $fh = IO::File->new($filename, "w");
4925     return undef if !$fh;
4926
4927     my $res = print $fh $buf;
4928
4929     $fh->close();
4930
4931     return $res;
4932 }
4933
4934 sub pci_device_info {
4935     my ($name) = @_;
4936
4937     my $res;
4938
4939     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4940     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4941
4942     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4943     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4944
4945     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4946     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4947
4948     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4949     return undef if !defined($product) || $product !~ s/^0x//;
4950
4951     $res = {
4952         name => $name,
4953         vendor => $vendor,
4954         product => $product,
4955         domain => $domain,
4956         bus => $bus,
4957         slot => $slot,
4958         func => $func,
4959         irq => $irq,
4960         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4961     };
4962
4963     return $res;
4964 }
4965
4966 sub pci_dev_reset {
4967     my ($dev) = @_;
4968
4969     my $name = $dev->{name};
4970
4971     my $fn = "$pcisysfs/devices/$name/reset";
4972
4973     return file_write($fn, "1");
4974 }
4975
4976 sub pci_dev_bind_to_vfio {
4977     my ($dev) = @_;
4978
4979     my $name = $dev->{name};
4980
4981     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4982
4983     if (!-d $vfio_basedir) {
4984         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4985     }
4986     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4987
4988     my $testdir = "$vfio_basedir/$name";
4989     return 1 if -d $testdir;
4990
4991     my $data = "$dev->{vendor} $dev->{product}";
4992     return undef if !file_write("$vfio_basedir/new_id", $data);
4993
4994     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4995     if (!file_write($fn, $name)) {
4996         return undef if -f $fn;
4997     }
4998
4999     $fn = "$vfio_basedir/bind";
5000     if (! -d $testdir) {
5001         return undef if !file_write($fn, $name);
5002     }
5003
5004     return -d $testdir;
5005 }
5006
5007 sub pci_dev_group_bind_to_vfio {
5008     my ($pciid) = @_;
5009
5010     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5011
5012     if (!-d $vfio_basedir) {
5013         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5014     }
5015     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5016
5017     # get IOMMU group devices
5018     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5019       my @devs = grep /^0000:/, readdir($D);
5020     closedir($D);
5021
5022     foreach my $pciid (@devs) {
5023         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5024
5025         # pci bridges, switches or root ports are not supported
5026         # they have a pci_bus subdirectory so skip them
5027         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5028
5029         my $info = pci_device_info($1);
5030         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5031     }
5032
5033     return 1;
5034 }
5035
5036 # vzdump restore implementaion
5037
5038 sub tar_archive_read_firstfile {
5039     my $archive = shift;
5040
5041     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5042
5043     # try to detect archive type first
5044     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5045         die "unable to open file '$archive'\n";
5046     my $firstfile = <$fh>;
5047     kill 15, $pid;
5048     close $fh;
5049
5050     die "ERROR: archive contaions no data\n" if !$firstfile;
5051     chomp $firstfile;
5052
5053     return $firstfile;
5054 }
5055
5056 sub tar_restore_cleanup {
5057     my ($storecfg, $statfile) = @_;
5058
5059     print STDERR "starting cleanup\n";
5060
5061     if (my $fd = IO::File->new($statfile, "r")) {
5062         while (defined(my $line = <$fd>)) {
5063             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5064                 my $volid = $2;
5065                 eval {
5066                     if ($volid =~ m|^/|) {
5067                         unlink $volid || die 'unlink failed\n';
5068                     } else {
5069                         PVE::Storage::vdisk_free($storecfg, $volid);
5070                     }
5071                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5072                 };
5073                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5074             } else {
5075                 print STDERR "unable to parse line in statfile - $line";
5076             }
5077         }
5078         $fd->close();
5079     }
5080 }
5081
5082 sub restore_archive {
5083     my ($archive, $vmid, $user, $opts) = @_;
5084
5085     my $format = $opts->{format};
5086     my $comp;
5087
5088     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5089         $format = 'tar' if !$format;
5090         $comp = 'gzip';
5091     } elsif ($archive =~ m/\.tar$/) {
5092         $format = 'tar' if !$format;
5093     } elsif ($archive =~ m/.tar.lzo$/) {
5094         $format = 'tar' if !$format;
5095         $comp = 'lzop';
5096     } elsif ($archive =~ m/\.vma$/) {
5097         $format = 'vma' if !$format;
5098     } elsif ($archive =~ m/\.vma\.gz$/) {
5099         $format = 'vma' if !$format;
5100         $comp = 'gzip';
5101     } elsif ($archive =~ m/\.vma\.lzo$/) {
5102         $format = 'vma' if !$format;
5103         $comp = 'lzop';
5104     } else {
5105         $format = 'vma' if !$format; # default
5106     }
5107
5108     # try to detect archive format
5109     if ($format eq 'tar') {
5110         return restore_tar_archive($archive, $vmid, $user, $opts);
5111     } else {
5112         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5113     }
5114 }
5115
5116 sub restore_update_config_line {
5117     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5118
5119     return if $line =~ m/^\#qmdump\#/;
5120     return if $line =~ m/^\#vzdump\#/;
5121     return if $line =~ m/^lock:/;
5122     return if $line =~ m/^unused\d+:/;
5123     return if $line =~ m/^parent:/;
5124     return if $line =~ m/^template:/; # restored VM is never a template
5125
5126     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5127     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5128         # try to convert old 1.X settings
5129         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5130         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5131             my ($model, $macaddr) = split(/\=/, $devconfig);
5132             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5133             my $net = {
5134                 model => $model,
5135                 bridge => "vmbr$ind",
5136                 macaddr => $macaddr,
5137             };
5138             my $netstr = print_net($net);
5139
5140             print $outfd "net$cookie->{netcount}: $netstr\n";
5141             $cookie->{netcount}++;
5142         }
5143     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5144         my ($id, $netstr) = ($1, $2);
5145         my $net = parse_net($netstr);
5146         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5147         $netstr = print_net($net);
5148         print $outfd "$id: $netstr\n";
5149     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5150         my $virtdev = $1;
5151         my $value = $3;
5152         my $di = parse_drive($virtdev, $value);
5153         if (defined($di->{backup}) && !$di->{backup}) {
5154             print $outfd "#$line";
5155         } elsif ($map->{$virtdev}) {
5156             delete $di->{format}; # format can change on restore
5157             $di->{file} = $map->{$virtdev};
5158             $value = print_drive($vmid, $di);
5159             print $outfd "$virtdev: $value\n";
5160         } else {
5161             print $outfd $line;
5162         }
5163     } else {
5164         print $outfd $line;
5165     }
5166 }
5167
5168 sub scan_volids {
5169     my ($cfg, $vmid) = @_;
5170
5171     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5172
5173     my $volid_hash = {};
5174     foreach my $storeid (keys %$info) {
5175         foreach my $item (@{$info->{$storeid}}) {
5176             next if !($item->{volid} && $item->{size});
5177             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5178             $volid_hash->{$item->{volid}} = $item;
5179         }
5180     }
5181
5182     return $volid_hash;
5183 }
5184
5185 sub is_volume_in_use {
5186     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5187
5188     my $path = PVE::Storage::path($storecfg, $volid);
5189
5190     my $scan_config = sub {
5191         my ($cref, $snapname) = @_;
5192
5193         foreach my $key (keys %$cref) {
5194             my $value = $cref->{$key};
5195             if (is_valid_drivename($key)) {
5196                 next if $skip_drive && $key eq $skip_drive;
5197                 my $drive = parse_drive($key, $value);
5198                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5199                 return 1 if $volid eq $drive->{file};
5200                 if ($drive->{file} =~ m!^/!) {
5201                     return 1 if $drive->{file} eq $path;
5202                 } else {
5203                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5204                     next if !$storeid;
5205                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5206                     next if !$scfg;
5207                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5208                 }
5209             }
5210         }
5211
5212         return 0;
5213     };
5214
5215     return 1 if &$scan_config($conf);
5216
5217     undef $skip_drive;
5218
5219     foreach my $snapname (keys %{$conf->{snapshots}}) {
5220         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5221     }
5222
5223     return 0;
5224 }
5225
5226 sub update_disksize {
5227     my ($vmid, $conf, $volid_hash) = @_;
5228
5229     my $changes;
5230
5231     my $used = {};
5232
5233     # Note: it is allowed to define multiple storages with same path (alias), so
5234     # we need to check both 'volid' and real 'path' (two different volid can point
5235     # to the same path).
5236
5237     my $usedpath = {};
5238
5239     # update size info
5240     foreach my $opt (keys %$conf) {
5241         if (is_valid_drivename($opt)) {
5242             my $drive = parse_drive($opt, $conf->{$opt});
5243             my $volid = $drive->{file};
5244             next if !$volid;
5245
5246             $used->{$volid} = 1;
5247             if ($volid_hash->{$volid} &&
5248                 (my $path = $volid_hash->{$volid}->{path})) {
5249                 $usedpath->{$path} = 1;
5250             }
5251
5252             next if drive_is_cdrom($drive);
5253             next if !$volid_hash->{$volid};
5254
5255             $drive->{size} = $volid_hash->{$volid}->{size};
5256             my $new = print_drive($vmid, $drive);
5257             if ($new ne $conf->{$opt}) {
5258                 $changes = 1;
5259                 $conf->{$opt} = $new;
5260             }
5261         }
5262     }
5263
5264     # remove 'unusedX' entry if volume is used
5265     foreach my $opt (keys %$conf) {
5266         next if $opt !~ m/^unused\d+$/;
5267         my $volid = $conf->{$opt};
5268         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5269         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5270             $changes = 1;
5271             delete $conf->{$opt};
5272         }
5273     }
5274
5275     foreach my $volid (sort keys %$volid_hash) {
5276         next if $volid =~ m/vm-$vmid-state-/;
5277         next if $used->{$volid};
5278         my $path = $volid_hash->{$volid}->{path};
5279         next if !$path; # just to be sure
5280         next if $usedpath->{$path};
5281         $changes = 1;
5282         PVE::QemuConfig->add_unused_volume($conf, $volid);
5283         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5284     }
5285
5286     return $changes;
5287 }
5288
5289 sub rescan {
5290     my ($vmid, $nolock) = @_;
5291
5292     my $cfg = PVE::Storage::config();
5293
5294     my $volid_hash = scan_volids($cfg, $vmid);
5295
5296     my $updatefn =  sub {
5297         my ($vmid) = @_;
5298
5299         my $conf = PVE::QemuConfig->load_config($vmid);
5300
5301         PVE::QemuConfig->check_lock($conf);
5302
5303         my $vm_volids = {};
5304         foreach my $volid (keys %$volid_hash) {
5305             my $info = $volid_hash->{$volid};
5306             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5307         }
5308
5309         my $changes = update_disksize($vmid, $conf, $vm_volids);
5310
5311         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5312     };
5313
5314     if (defined($vmid)) {
5315         if ($nolock) {
5316             &$updatefn($vmid);
5317         } else {
5318             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5319         }
5320     } else {
5321         my $vmlist = config_list();
5322         foreach my $vmid (keys %$vmlist) {
5323             if ($nolock) {
5324                 &$updatefn($vmid);
5325             } else {
5326                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5327             }
5328         }
5329     }
5330 }
5331
5332 sub restore_vma_archive {
5333     my ($archive, $vmid, $user, $opts, $comp) = @_;
5334
5335     my $input = $archive eq '-' ? "<&STDIN" : undef;
5336     my $readfrom = $archive;
5337
5338     my $uncomp = '';
5339     if ($comp) {
5340         $readfrom = '-';
5341         my $qarchive = PVE::Tools::shellquote($archive);
5342         if ($comp eq 'gzip') {
5343             $uncomp = "zcat $qarchive|";
5344         } elsif ($comp eq 'lzop') {
5345             $uncomp = "lzop -d -c $qarchive|";
5346         } else {
5347             die "unknown compression method '$comp'\n";
5348         }
5349
5350     }
5351
5352     my $tmpdir = "/var/tmp/vzdumptmp$$";
5353     rmtree $tmpdir;
5354
5355     # disable interrupts (always do cleanups)
5356     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5357         warn "got interrupt - ignored\n";
5358     };
5359
5360     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5361     POSIX::mkfifo($mapfifo, 0600);
5362     my $fifofh;
5363
5364     my $openfifo = sub {
5365         open($fifofh, '>', $mapfifo) || die $!;
5366     };
5367
5368     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5369
5370     my $oldtimeout;
5371     my $timeout = 5;
5372
5373     my $devinfo = {};
5374
5375     my $rpcenv = PVE::RPCEnvironment::get();
5376
5377     my $conffile = PVE::QemuConfig->config_file($vmid);
5378     my $tmpfn = "$conffile.$$.tmp";
5379
5380     # Note: $oldconf is undef if VM does not exists
5381     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5382     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5383
5384     my $print_devmap = sub {
5385         my $virtdev_hash = {};
5386
5387         my $cfgfn = "$tmpdir/qemu-server.conf";
5388
5389         # we can read the config - that is already extracted
5390         my $fh = IO::File->new($cfgfn, "r") ||
5391             "unable to read qemu-server.conf - $!\n";
5392
5393         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5394         if (-f $fwcfgfn) {
5395             my $pve_firewall_dir = '/etc/pve/firewall';
5396             mkdir $pve_firewall_dir; # make sure the dir exists
5397             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5398         }
5399
5400         while (defined(my $line = <$fh>)) {
5401             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5402                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5403                 die "archive does not contain data for drive '$virtdev'\n"
5404                     if !$devinfo->{$devname};
5405                 if (defined($opts->{storage})) {
5406                     $storeid = $opts->{storage} || 'local';
5407                 } elsif (!$storeid) {
5408                     $storeid = 'local';
5409                 }
5410                 $format = 'raw' if !$format;
5411                 $devinfo->{$devname}->{devname} = $devname;
5412                 $devinfo->{$devname}->{virtdev} = $virtdev;
5413                 $devinfo->{$devname}->{format} = $format;
5414                 $devinfo->{$devname}->{storeid} = $storeid;
5415
5416                 # check permission on storage
5417                 my $pool = $opts->{pool}; # todo: do we need that?
5418                 if ($user ne 'root@pam') {
5419                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5420                 }
5421
5422                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5423             }
5424         }
5425
5426         foreach my $devname (keys %$devinfo) {
5427             die "found no device mapping information for device '$devname'\n"
5428                 if !$devinfo->{$devname}->{virtdev};
5429         }
5430
5431         my $cfg = PVE::Storage::config();
5432
5433         # create empty/temp config
5434         if ($oldconf) {
5435             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5436             foreach_drive($oldconf, sub {
5437                 my ($ds, $drive) = @_;
5438
5439                 return if drive_is_cdrom($drive);
5440
5441                 my $volid = $drive->{file};
5442
5443                 return if !$volid || $volid =~ m|^/|;
5444
5445                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5446                 return if !$path || !$owner || ($owner != $vmid);
5447
5448                 # Note: only delete disk we want to restore
5449                 # other volumes will become unused
5450                 if ($virtdev_hash->{$ds}) {
5451                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5452                     if (my $err = $@) {
5453                         warn $err;
5454                     }
5455                 }
5456             });
5457
5458             # delete vmstate files
5459             # since after the restore we have no snapshots anymore
5460             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5461                 my $snap = $oldconf->{snapshots}->{$snapname};
5462                 if ($snap->{vmstate}) {
5463                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5464                     if (my $err = $@) {
5465                         warn $err;
5466                     }
5467                 }
5468             }
5469         }
5470
5471         my $map = {};
5472         foreach my $virtdev (sort keys %$virtdev_hash) {
5473             my $d = $virtdev_hash->{$virtdev};
5474             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5475             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5476
5477             # test if requested format is supported
5478             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5479             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5480             $d->{format} = $defFormat if !$supported;
5481
5482             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5483                                                   $d->{format}, undef, $alloc_size);
5484             print STDERR "new volume ID is '$volid'\n";
5485             $d->{volid} = $volid;
5486             my $path = PVE::Storage::path($cfg, $volid);
5487
5488             PVE::Storage::activate_volumes($cfg,[$volid]);
5489
5490             my $write_zeros = 1;
5491             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5492                 $write_zeros = 0;
5493             }
5494
5495             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5496
5497             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5498             $map->{$virtdev} = $volid;
5499         }
5500
5501         $fh->seek(0, 0) || die "seek failed - $!\n";
5502
5503         my $outfd = new IO::File ($tmpfn, "w") ||
5504             die "unable to write config for VM $vmid\n";
5505
5506         my $cookie = { netcount => 0 };
5507         while (defined(my $line = <$fh>)) {
5508             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5509         }
5510
5511         $fh->close();
5512         $outfd->close();
5513     };
5514
5515     eval {
5516         # enable interrupts
5517         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5518             die "interrupted by signal\n";
5519         };
5520         local $SIG{ALRM} = sub { die "got timeout\n"; };
5521
5522         $oldtimeout = alarm($timeout);
5523
5524         my $parser = sub {
5525             my $line = shift;
5526
5527             print "$line\n";
5528
5529             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5530                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5531                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5532             } elsif ($line =~ m/^CTIME: /) {
5533                 # we correctly received the vma config, so we can disable
5534                 # the timeout now for disk allocation (set to 10 minutes, so
5535                 # that we always timeout if something goes wrong)
5536                 alarm(600);
5537                 &$print_devmap();
5538                 print $fifofh "done\n";
5539                 my $tmp = $oldtimeout || 0;
5540                 $oldtimeout = undef;
5541                 alarm($tmp);
5542                 close($fifofh);
5543             }
5544         };
5545
5546         print "restore vma archive: $cmd\n";
5547         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5548     };
5549     my $err = $@;
5550
5551     alarm($oldtimeout) if $oldtimeout;
5552
5553     my $vollist = [];
5554     foreach my $devname (keys %$devinfo) {
5555         my $volid = $devinfo->{$devname}->{volid};
5556         push @$vollist, $volid if $volid;
5557     }
5558
5559     my $cfg = PVE::Storage::config();
5560     PVE::Storage::deactivate_volumes($cfg, $vollist);
5561
5562     unlink $mapfifo;
5563
5564     if ($err) {
5565         rmtree $tmpdir;
5566         unlink $tmpfn;
5567
5568         foreach my $devname (keys %$devinfo) {
5569             my $volid = $devinfo->{$devname}->{volid};
5570             next if !$volid;
5571             eval {
5572                 if ($volid =~ m|^/|) {
5573                     unlink $volid || die 'unlink failed\n';
5574                 } else {
5575                     PVE::Storage::vdisk_free($cfg, $volid);
5576                 }
5577                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5578             };
5579             print STDERR "unable to cleanup '$volid' - $@" if $@;
5580         }
5581         die $err;
5582     }
5583
5584     rmtree $tmpdir;
5585
5586     rename($tmpfn, $conffile) ||
5587         die "unable to commit configuration file '$conffile'\n";
5588
5589     PVE::Cluster::cfs_update(); # make sure we read new file
5590
5591     eval { rescan($vmid, 1); };
5592     warn $@ if $@;
5593 }
5594
5595 sub restore_tar_archive {
5596     my ($archive, $vmid, $user, $opts) = @_;
5597
5598     if ($archive ne '-') {
5599         my $firstfile = tar_archive_read_firstfile($archive);
5600         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5601             if $firstfile ne 'qemu-server.conf';
5602     }
5603
5604     my $storecfg = PVE::Storage::config();
5605
5606     # destroy existing data - keep empty config
5607     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5608     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5609
5610     my $tocmd = "/usr/lib/qemu-server/qmextract";
5611
5612     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5613     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5614     $tocmd .= ' --prealloc' if $opts->{prealloc};
5615     $tocmd .= ' --info' if $opts->{info};
5616
5617     # tar option "xf" does not autodetect compression when read from STDIN,
5618     # so we pipe to zcat
5619     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5620         PVE::Tools::shellquote("--to-command=$tocmd");
5621
5622     my $tmpdir = "/var/tmp/vzdumptmp$$";
5623     mkpath $tmpdir;
5624
5625     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5626     local $ENV{VZDUMP_VMID} = $vmid;
5627     local $ENV{VZDUMP_USER} = $user;
5628
5629     my $conffile = PVE::QemuConfig->config_file($vmid);
5630     my $tmpfn = "$conffile.$$.tmp";
5631
5632     # disable interrupts (always do cleanups)
5633     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5634         print STDERR "got interrupt - ignored\n";
5635     };
5636
5637     eval {
5638         # enable interrupts
5639         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5640             die "interrupted by signal\n";
5641         };
5642
5643         if ($archive eq '-') {
5644             print "extracting archive from STDIN\n";
5645             run_command($cmd, input => "<&STDIN");
5646         } else {
5647             print "extracting archive '$archive'\n";
5648             run_command($cmd);
5649         }
5650
5651         return if $opts->{info};
5652
5653         # read new mapping
5654         my $map = {};
5655         my $statfile = "$tmpdir/qmrestore.stat";
5656         if (my $fd = IO::File->new($statfile, "r")) {
5657             while (defined (my $line = <$fd>)) {
5658                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5659                     $map->{$1} = $2 if $1;
5660                 } else {
5661                     print STDERR "unable to parse line in statfile - $line\n";
5662                 }
5663             }
5664             $fd->close();
5665         }
5666
5667         my $confsrc = "$tmpdir/qemu-server.conf";
5668
5669         my $srcfd = new IO::File($confsrc, "r") ||
5670             die "unable to open file '$confsrc'\n";
5671
5672         my $outfd = new IO::File ($tmpfn, "w") ||
5673             die "unable to write config for VM $vmid\n";
5674
5675         my $cookie = { netcount => 0 };
5676         while (defined (my $line = <$srcfd>)) {
5677             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5678         }
5679
5680         $srcfd->close();
5681         $outfd->close();
5682     };
5683     my $err = $@;
5684
5685     if ($err) {
5686
5687         unlink $tmpfn;
5688
5689         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5690
5691         die $err;
5692     }
5693
5694     rmtree $tmpdir;
5695
5696     rename $tmpfn, $conffile ||
5697         die "unable to commit configuration file '$conffile'\n";
5698
5699     PVE::Cluster::cfs_update(); # make sure we read new file
5700
5701     eval { rescan($vmid, 1); };
5702     warn $@ if $@;
5703 };
5704
5705 sub foreach_writable_storage {
5706     my ($conf, $func) = @_;
5707
5708     my $sidhash = {};
5709
5710     foreach my $ds (keys %$conf) {
5711         next if !is_valid_drivename($ds);
5712
5713         my $drive = parse_drive($ds, $conf->{$ds});
5714         next if !$drive;
5715         next if drive_is_cdrom($drive);
5716
5717         my $volid = $drive->{file};
5718
5719         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5720         $sidhash->{$sid} = $sid if $sid;
5721     }
5722
5723     foreach my $sid (sort keys %$sidhash) {
5724         &$func($sid);
5725     }
5726 }
5727
5728 sub do_snapshots_with_qemu {
5729     my ($storecfg, $volid) = @_;
5730
5731     my $storage_name = PVE::Storage::parse_volume_id($volid);
5732
5733     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5734         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5735         return 1;
5736     }
5737
5738     if ($volid =~ m/\.(qcow2|qed)$/){
5739         return 1;
5740     }
5741
5742     return undef;
5743 }
5744
5745 sub qga_check_running {
5746     my ($vmid) = @_;
5747
5748     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5749     if ($@) {
5750         warn "Qemu Guest Agent are not running - $@";
5751         return 0;
5752     }
5753     return 1;
5754 }
5755
5756 sub template_create {
5757     my ($vmid, $conf, $disk) = @_;
5758
5759     my $storecfg = PVE::Storage::config();
5760
5761     foreach_drive($conf, sub {
5762         my ($ds, $drive) = @_;
5763
5764         return if drive_is_cdrom($drive);
5765         return if $disk && $ds ne $disk;
5766
5767         my $volid = $drive->{file};
5768         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5769
5770         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5771         $drive->{file} = $voliddst;
5772         $conf->{$ds} = print_drive($vmid, $drive);
5773         PVE::QemuConfig->write_config($vmid, $conf);
5774     });
5775 }
5776
5777 sub qemu_img_convert {
5778     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5779
5780     my $storecfg = PVE::Storage::config();
5781     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5782     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5783
5784     if ($src_storeid && $dst_storeid) {
5785
5786         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5787
5788         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5789         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5790
5791         my $src_format = qemu_img_format($src_scfg, $src_volname);
5792         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5793
5794         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5795         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5796
5797         my $cmd = [];
5798         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5799         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5800         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5801         if ($is_zero_initialized) {
5802             push @$cmd, "zeroinit:$dst_path";
5803         } else {
5804             push @$cmd, $dst_path;
5805         }
5806
5807         my $parser = sub {
5808             my $line = shift;
5809             if($line =~ m/\((\S+)\/100\%\)/){
5810                 my $percent = $1;
5811                 my $transferred = int($size * $percent / 100);
5812                 my $remaining = $size - $transferred;
5813
5814                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5815             }
5816
5817         };
5818
5819         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5820         my $err = $@;
5821         die "copy failed: $err" if $err;
5822     }
5823 }
5824
5825 sub qemu_img_format {
5826     my ($scfg, $volname) = @_;
5827
5828     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5829         return $1;
5830     } else {
5831         return "raw";
5832     }
5833 }
5834
5835 sub qemu_drive_mirror {
5836     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5837
5838     my $storecfg = PVE::Storage::config();
5839     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5840
5841     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5842
5843     my $format = qemu_img_format($dst_scfg, $dst_volname);
5844
5845     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5846
5847     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5848
5849     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5850     $opts->{format} = $format if $format;
5851
5852     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5853
5854     my $finish_job = sub {
5855         while (1) {
5856             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5857             my $stat = @$stats[0];
5858             last if !$stat;
5859             sleep 1;
5860         }
5861     };
5862
5863     eval {
5864     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5865         while (1) {
5866             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5867             my $stat = @$stats[0];
5868             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5869             die "error job is not mirroring" if $stat->{type} ne "mirror";
5870
5871             my $busy = $stat->{busy};
5872             my $ready = $stat->{ready};
5873
5874             if (my $total = $stat->{len}) {
5875                 my $transferred = $stat->{offset} || 0;
5876                 my $remaining = $total - $transferred;
5877                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5878
5879                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5880             }
5881
5882
5883             if ($stat->{ready} eq 'true') {
5884
5885                 last if $vmiddst != $vmid;
5886
5887                 # try to switch the disk if source and destination are on the same guest
5888                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5889                 if (!$@) {
5890                     &$finish_job();
5891                     last;
5892                 }
5893                 die $@ if $@ !~ m/cannot be completed/;
5894             }
5895             sleep 1;
5896         }
5897
5898
5899     };
5900     my $err = $@;
5901
5902     my $cancel_job = sub {
5903         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5904         &$finish_job();
5905     };
5906
5907     if ($err) {
5908         eval { &$cancel_job(); };
5909         die "mirroring error: $err";
5910     }
5911
5912     if ($vmiddst != $vmid) {
5913         # if we clone a disk for a new target vm, we don't switch the disk
5914         &$cancel_job(); # so we call block-job-cancel
5915     }
5916 }
5917
5918 sub clone_disk {
5919     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5920         $newvmid, $storage, $format, $full, $newvollist) = @_;
5921
5922     my $newvolid;
5923
5924     if (!$full) {
5925         print "create linked clone of drive $drivename ($drive->{file})\n";
5926         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5927         push @$newvollist, $newvolid;
5928     } else {
5929         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5930         $storeid = $storage if $storage;
5931
5932         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5933         if (!$format) {
5934             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5935             $format = qemu_img_format($scfg, $volname);
5936         }
5937
5938         # test if requested format is supported - else use default
5939         my $supported = grep { $_ eq $format } @$validFormats;
5940         $format = $defFormat if !$supported;
5941
5942         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5943
5944         print "create full clone of drive $drivename ($drive->{file})\n";
5945         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5946         push @$newvollist, $newvolid;
5947
5948         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
5949
5950         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
5951         if (!$running || $snapname) {
5952             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
5953         } else {
5954
5955             my $kvmver = get_running_qemu_version ($vmid);
5956             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
5957                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
5958                     if $drive->{iothread};
5959             }
5960
5961             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
5962         }
5963     }
5964
5965     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5966
5967     my $disk = $drive;
5968     $disk->{format} = undef;
5969     $disk->{file} = $newvolid;
5970     $disk->{size} = $size;
5971
5972     return $disk;
5973 }
5974
5975 # this only works if VM is running
5976 sub get_current_qemu_machine {
5977     my ($vmid) = @_;
5978
5979     my $cmd = { execute => 'query-machines', arguments => {} };
5980     my $res = vm_qmp_command($vmid, $cmd);
5981
5982     my ($current, $default);
5983     foreach my $e (@$res) {
5984         $default = $e->{name} if $e->{'is-default'};
5985         $current = $e->{name} if $e->{'is-current'};
5986     }
5987
5988     # fallback to the default machine if current is not supported by qemu
5989     return $current || $default || 'pc';
5990 }
5991
5992 sub get_running_qemu_version {
5993     my ($vmid) = @_;
5994     my $cmd = { execute => 'query-version', arguments => {} };
5995     my $res = vm_qmp_command($vmid, $cmd);
5996     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
5997 }
5998
5999 sub qemu_machine_feature_enabled {
6000     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6001
6002     my $current_major;
6003     my $current_minor;
6004
6005     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6006
6007         $current_major = $3;
6008         $current_minor = $4;
6009
6010     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6011
6012         $current_major = $1;
6013         $current_minor = $2;
6014     }
6015
6016     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6017
6018
6019 }
6020
6021 sub qemu_machine_pxe {
6022     my ($vmid, $conf, $machine) = @_;
6023
6024     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6025
6026     foreach my $opt (keys %$conf) {
6027         next if $opt !~ m/^net(\d+)$/;
6028         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6029         next if !$net;
6030         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6031         return $machine.".pxe" if $romfile =~ m/pxe/;
6032         last;
6033     }
6034
6035     return $machine;
6036 }
6037
6038 sub qemu_use_old_bios_files {
6039     my ($machine_type) = @_;
6040
6041     return if !$machine_type;
6042
6043     my $use_old_bios_files = undef;
6044
6045     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6046         $machine_type = $1;
6047         $use_old_bios_files = 1;
6048     } else {
6049         my $kvmver = kvm_user_version();
6050         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6051         # load new efi bios files on migration. So this hack is required to allow
6052         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6053         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6054         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6055     }
6056
6057     return ($use_old_bios_files, $machine_type);
6058 }
6059
6060 sub lspci {
6061
6062     my $devices = {};
6063
6064     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6065             my (undef, $id, $function) = @_;
6066             my $res = { id => $id, function => $function};
6067             push @{$devices->{$id}}, $res;
6068     });
6069
6070     return $devices;
6071 }
6072
6073 sub vm_iothreads_list {
6074     my ($vmid) = @_;
6075
6076     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6077
6078     my $iothreads = {};
6079     foreach my $iothread (@$res) {
6080         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6081     }
6082
6083     return $iothreads;
6084 }
6085
6086 sub scsihw_infos {
6087     my ($conf, $drive) = @_;
6088
6089     my $maxdev = 0;
6090
6091     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6092         $maxdev = 7;
6093     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6094         $maxdev = 1;
6095     } else {
6096         $maxdev = 256;
6097     }
6098
6099     my $controller = int($drive->{index} / $maxdev);
6100     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6101
6102     return ($maxdev, $controller, $controller_prefix);
6103 }
6104
6105 sub add_hyperv_enlighments {
6106     my ($cpuFlags, $winversion, $machine_type, $kvmver, $nokvm, $bios, $gpu_passthrough) = @_;
6107
6108     return if $nokvm;
6109     return if $winversion < 6;
6110     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6111
6112     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6113
6114     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6115         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6116         push @$cpuFlags , 'hv_vapic';
6117         push @$cpuFlags , 'hv_time';
6118     } else {
6119         push @$cpuFlags , 'hv_spinlocks=0xffff';
6120     }
6121
6122     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6123         push @$cpuFlags , 'hv_reset';
6124         push @$cpuFlags , 'hv_vpindex';
6125         push @$cpuFlags , 'hv_runtime';
6126     }
6127
6128     if ($winversion >= 7) {
6129         push @$cpuFlags , 'hv_relaxed';
6130     }
6131 }
6132
6133 sub windows_version {
6134     my ($ostype) = @_;
6135
6136     return 0 if !$ostype;
6137
6138     my $winversion = 0;
6139
6140     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6141         $winversion = 5;
6142     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6143         $winversion = 6;
6144     } elsif ($ostype =~ m/^win(\d+)$/) {
6145         $winversion = $1;
6146     }
6147
6148     return $winversion;
6149 }
6150
6151 # bash completion helper
6152
6153 sub complete_backup_archives {
6154     my ($cmdname, $pname, $cvalue) = @_;
6155
6156     my $cfg = PVE::Storage::config();
6157
6158     my $storeid;
6159
6160     if ($cvalue =~ m/^([^:]+):/) {
6161         $storeid = $1;
6162     }
6163
6164     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6165
6166     my $res = [];
6167     foreach my $id (keys %$data) {
6168         foreach my $item (@{$data->{$id}}) {
6169             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6170             push @$res, $item->{volid} if defined($item->{volid});
6171         }
6172     }
6173
6174     return $res;
6175 }
6176
6177 my $complete_vmid_full = sub {
6178     my ($running) = @_;
6179
6180     my $idlist = vmstatus();
6181
6182     my $res = [];
6183
6184     foreach my $id (keys %$idlist) {
6185         my $d = $idlist->{$id};
6186         if (defined($running)) {
6187             next if $d->{template};
6188             next if $running && $d->{status} ne 'running';
6189             next if !$running && $d->{status} eq 'running';
6190         }
6191         push @$res, $id;
6192
6193     }
6194     return $res;
6195 };
6196
6197 sub complete_vmid {
6198     return &$complete_vmid_full();
6199 }
6200
6201 sub complete_vmid_stopped {
6202     return &$complete_vmid_full(0);
6203 }
6204
6205 sub complete_vmid_running {
6206     return &$complete_vmid_full(1);
6207 }
6208
6209 sub complete_storage {
6210
6211     my $cfg = PVE::Storage::config();
6212     my $ids = $cfg->{ids};
6213
6214     my $res = [];
6215     foreach my $sid (keys %$ids) {
6216         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6217         next if !$ids->{$sid}->{content}->{images};
6218         push @$res, $sid;
6219     }
6220
6221     return $res;
6222 }
6223
6224 1;