PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42 my $OVMF_IMG = '/usr/share/kvm/OVMF-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 # Note about locking: we use flock on the config file protect
  49 # against concurent actions.
  50 # Aditionaly, we have a 'lock' setting in the config file. This
  51 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  52 # allowed when such lock is set. But you can ignore this kind of
  53 # lock with the --skiplock flag.
  54
  55 cfs_register_file('/qemu-server/',
  56                   \&parse_vm_config,
  57                   \&write_vm_config);
  58
  59 PVE::JSONSchema::register_standard_option('skiplock', {
  60     description => "Ignore locks - only root is allowed to use this option.",
  61     type => 'boolean',
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  66     description => "Some command save/restore state from this location.",
  67     type => 'string',
  68     maxLength => 128,
  69     optional => 1,
  70 });
  71
  72 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  73     description => "The name of the snapshot.",
  74     type => 'string', format => 'pve-configid',
  75     maxLength => 40,
  76 });
  77
  78 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  79     type => 'string',
  80     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  81     description => "The drive's backing file's data format.",
  82     optional => 1,
  83 });
  84
  85 #no warnings 'redefine';
  86
  87 sub cgroups_write {
  88    my ($controller, $vmid, $option, $value) = @_;
  89
  90    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  91    PVE::ProcFSTools::write_proc_entry($path, $value);
  92
  93 }
  94
  95 my $nodename = PVE::INotify::nodename();
  96
  97 mkdir "/etc/pve/nodes/$nodename";
  98 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  99 mkdir $confdir;
 100
 101 my $var_run_tmpdir = "/var/run/qemu-server";
 102 mkdir $var_run_tmpdir;
 103
 104 my $lock_dir = "/var/lock/qemu-server";
 105 mkdir $lock_dir;
 106
 107 my $pcisysfs = "/sys/bus/pci";
 108
 109 my $cpu_vendor_list = {
 110     # Intel CPUs
 111     486 => 'GenuineIntel',
 112     pentium => 'GenuineIntel',
 113     pentium2  => 'GenuineIntel',
 114     pentium3  => 'GenuineIntel',
 115     coreduo => 'GenuineIntel',
 116     core2duo => 'GenuineIntel',
 117     Conroe  => 'GenuineIntel',
 118     Penryn  => 'GenuineIntel',
 119     Nehalem  => 'GenuineIntel',
 120     Westmere => 'GenuineIntel',
 121     SandyBridge => 'GenuineIntel',
 122     IvyBridge => 'GenuineIntel',
 123     Haswell => 'GenuineIntel',
 124     'Haswell-noTSX' => 'GenuineIntel',
 125     Broadwell => 'GenuineIntel',
 126     'Broadwell-noTSX' => 'GenuineIntel',
 127
 128     # AMD CPUs
 129     athlon => 'AuthenticAMD',
 130     phenom  => 'AuthenticAMD',
 131     Opteron_G1  => 'AuthenticAMD',
 132     Opteron_G2  => 'AuthenticAMD',
 133     Opteron_G3  => 'AuthenticAMD',
 134     Opteron_G4  => 'AuthenticAMD',
 135     Opteron_G5  => 'AuthenticAMD',
 136
 137     # generic types, use vendor from host node
 138     host => 'default',
 139     kvm32 => 'default',
 140     kvm64 => 'default',
 141     qemu32 => 'default',
 142     qemu64 => 'default',
 143 };
 144
 145 my $cpu_fmt = {
 146     cputype => {
 147         description => "Emulated CPU type.",
 148         type => 'string',
 149         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 150         default => 'kvm64',
 151         default_key => 1,
 152     },
 153     hidden => {
 154         description => "Do not identify as a KVM virtual machine.",
 155         type => 'boolean',
 156         optional => 1,
 157         default => 0
 158     },
 159 };
 160
 161 my $watchdog_fmt = {
 162     model => {
 163         default_key => 1,
 164         type => 'string',
 165         enum => [qw(i6300esb ib700)],
 166         description => "Watchdog type to emulate.",
 167         default => 'i6300esb',
 168         optional => 1,
 169     },
 170     action => {
 171         type => 'string',
 172         enum => [qw(reset shutdown poweroff pause debug none)],
 173         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 174         optional => 1,
 175     },
 176 };
 177 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 178
 179 my $confdesc = {
 180     onboot => {
 181         optional => 1,
 182         type => 'boolean',
 183         description => "Specifies whether a VM will be started during system bootup.",
 184         default => 0,
 185     },
 186     autostart => {
 187         optional => 1,
 188         type => 'boolean',
 189         description => "Automatic restart after crash (currently ignored).",
 190         default => 0,
 191     },
 192     hotplug => {
 193         optional => 1,
 194         type => 'string', format => 'pve-hotplug-features',
 195         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 196         default => 'network,disk,usb',
 197     },
 198     reboot => {
 199         optional => 1,
 200         type => 'boolean',
 201         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 202         default => 1,
 203     },
 204     lock => {
 205         optional => 1,
 206         type => 'string',
 207         description => "Lock/unlock the VM.",
 208         enum => [qw(migrate backup snapshot rollback)],
 209     },
 210     cpulimit => {
 211         optional => 1,
 212         type => 'number',
 213         description => "Limit of CPU usage.",
 214         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 215         minimum => 0,
 216         maximum => 128,
 217         default => 0,
 218     },
 219     cpuunits => {
 220         optional => 1,
 221         type => 'integer',
 222         description => "CPU weight for a VM.",
 223         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 224         minimum => 0,
 225         maximum => 500000,
 226         default => 1024,
 227     },
 228     memory => {
 229         optional => 1,
 230         type => 'integer',
 231         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 232         minimum => 16,
 233         default => 512,
 234     },
 235     balloon => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 239         minimum => 0,
 240     },
 241     shares => {
 242         optional => 1,
 243         type => 'integer',
 244         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 245         minimum => 0,
 246         maximum => 50000,
 247         default => 1000,
 248     },
 249     keyboard => {
 250         optional => 1,
 251         type => 'string',
 252         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 253         enum => PVE::Tools::kvmkeymaplist(),
 254         default => 'en-us',
 255     },
 256     name => {
 257         optional => 1,
 258         type => 'string', format => 'dns-name',
 259         description => "Set a name for the VM. Only used on the configuration web interface.",
 260     },
 261     scsihw => {
 262         optional => 1,
 263         type => 'string',
 264         description => "SCSI controller model",
 265         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 266         default => 'lsi',
 267     },
 268     description => {
 269         optional => 1,
 270         type => 'string',
 271         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 272     },
 273     ostype => {
 274         optional => 1,
 275         type => 'string',
 276         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 l24 l26 solaris)],
 277         description => "Specify guest operating system.",
 278         verbose_description => <<EODESC,
 279 Specify guest operating system. This is used to enable special
 280 optimization/features for specific operating systems:
 281
 282 [horizontal]
 283 other;; unspecified OS
 284 wxp;; Microsoft Windows XP
 285 w2k;; Microsoft Windows 2000
 286 w2k3;; Microsoft Windows 2003
 287 w2k8;; Microsoft Windows 2008
 288 wvista;; Microsoft Windows Vista
 289 win7;; Microsoft Windows 7
 290 win8;; Microsoft Windows 8/2012
 291 l24;; Linux 2.4 Kernel
 292 l26;; Linux 2.6/3.X Kernel
 293 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 294 EODESC
 295     },
 296     boot => {
 297         optional => 1,
 298         type => 'string',
 299         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 300         pattern => '[acdn]{1,4}',
 301         default => 'cdn',
 302     },
 303     bootdisk => {
 304         optional => 1,
 305         type => 'string', format => 'pve-qm-bootdisk',
 306         description => "Enable booting from specified disk.",
 307         pattern => '(ide|sata|scsi|virtio)\d+',
 308     },
 309     smp => {
 310         optional => 1,
 311         type => 'integer',
 312         description => "The number of CPUs. Please use option -sockets instead.",
 313         minimum => 1,
 314         default => 1,
 315     },
 316     sockets => {
 317         optional => 1,
 318         type => 'integer',
 319         description => "The number of CPU sockets.",
 320         minimum => 1,
 321         default => 1,
 322     },
 323     cores => {
 324         optional => 1,
 325         type => 'integer',
 326         description => "The number of cores per socket.",
 327         minimum => 1,
 328         default => 1,
 329     },
 330     numa => {
 331         optional => 1,
 332         type => 'boolean',
 333         description => "Enable/disable NUMA.",
 334         default => 0,
 335     },
 336     hugepages => {
 337         optional => 1,
 338         type => 'string',
 339         description => "Enable/disable hugepages memory.",
 340         enum => [qw(any 2 1024)],
 341     },
 342     vcpus => {
 343         optional => 1,
 344         type => 'integer',
 345         description => "Number of hotplugged vcpus.",
 346         minimum => 1,
 347         default => 0,
 348     },
 349     acpi => {
 350         optional => 1,
 351         type => 'boolean',
 352         description => "Enable/disable ACPI.",
 353         default => 1,
 354     },
 355     agent => {
 356         optional => 1,
 357         type => 'boolean',
 358         description => "Enable/disable Qemu GuestAgent.",
 359         default => 0,
 360     },
 361     kvm => {
 362         optional => 1,
 363         type => 'boolean',
 364         description => "Enable/disable KVM hardware virtualization.",
 365         default => 1,
 366     },
 367     tdf => {
 368         optional => 1,
 369         type => 'boolean',
 370         description => "Enable/disable time drift fix.",
 371         default => 0,
 372     },
 373     localtime => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 377     },
 378     freeze => {
 379         optional => 1,
 380         type => 'boolean',
 381         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 382     },
 383     vga => {
 384         optional => 1,
 385         type => 'string',
 386         description => "Select the VGA type.",
 387         verbose_description => "Select the VGA type. If you want to use high resolution" .
 388             " modes (>= 1280x1024x16) then you should use the options " .
 389             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 390             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 391             "display sever. For win* OS you can select how many independent " .
 392             "displays you want, Linux guests can add displays them self. " .
 393             "You can also run without any graphic card, using a serial device" .
 394             " as terminal.",
 395         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 396     },
 397     watchdog => {
 398         optional => 1,
 399         type => 'string', format => 'pve-qm-watchdog',
 400         description => "Create a virtual hardware watchdog device.",
 401         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 402             " (by a guest action), the watchdog must be periodically polled " .
 403             "by an agent inside the guest or else the watchdog will reset " .
 404             "the guest (or execute the respective action specified)",
 405     },
 406     startdate => {
 407         optional => 1,
 408         type => 'string',
 409         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 410         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 411         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 412         default => 'now',
 413     },
 414     startup =>  get_standard_option('pve-startup-order'),
 415     template => {
 416         optional => 1,
 417         type => 'boolean',
 418         description => "Enable/disable Template.",
 419         default => 0,
 420     },
 421     args => {
 422         optional => 1,
 423         type => 'string',
 424         description => "Arbitrary arguments passed to kvm.",
 425         verbose_description => <<EODESCR,
 426 Arbitrary arguments passed to kvm, for example:
 427
 428 args: -no-reboot -no-hpet
 429
 430 NOTE: this option is for experts only.
 431 EODESCR
 432     },
 433     tablet => {
 434         optional => 1,
 435         type => 'boolean',
 436         default => 1,
 437         description => "Enable/disable the USB tablet device.",
 438         verbose_description => "Enable/disable the USB tablet device. This device is " .
 439             "usually needed to allow absolute mouse positioning with VNC. " .
 440             "Else the mouse runs out of sync with normal VNC clients. " .
 441             "If you're running lots of console-only guests on one host, " .
 442             "you may consider disabling this to save some context switches. " .
 443             "This is turned off by default if you use spice (-vga=qxl).",
 444     },
 445     migrate_speed => {
 446         optional => 1,
 447         type => 'integer',
 448         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 449         minimum => 0,
 450         default => 0,
 451     },
 452     migrate_downtime => {
 453         optional => 1,
 454         type => 'number',
 455         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 456         minimum => 0,
 457         default => 0.1,
 458     },
 459     cdrom => {
 460         optional => 1,
 461         type => 'string', format => 'pve-qm-ide',
 462         typetext => '<volume>',
 463         description => "This is an alias for option -ide2",
 464     },
 465     cpu => {
 466         optional => 1,
 467         description => "Emulated CPU type.",
 468         type => 'string',
 469         format => $cpu_fmt,
 470     },
 471     parent => get_standard_option('pve-snapshot-name', {
 472         optional => 1,
 473         description => "Parent snapshot name. This is used internally, and should not be modified.",
 474     }),
 475     snaptime => {
 476         optional => 1,
 477         description => "Timestamp for snapshots.",
 478         type => 'integer',
 479         minimum => 0,
 480     },
 481     vmstate => {
 482         optional => 1,
 483         type => 'string', format => 'pve-volume-id',
 484         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 485     },
 486     machine => {
 487         description => "Specific the Qemu machine type.",
 488         type => 'string',
 489         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 490         maxLength => 40,
 491         optional => 1,
 492     },
 493     smbios1 => {
 494         description => "Specify SMBIOS type 1 fields.",
 495         type => 'string', format => 'pve-qm-smbios1',
 496         maxLength => 256,
 497         optional => 1,
 498     },
 499     protection => {
 500         optional => 1,
 501         type => 'boolean',
 502         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 503         default => 0,
 504     },
 505     bios => {
 506         optional => 1,
 507         type => 'string',
 508         enum => [ qw(seabios ovmf) ],
 509         description => "Select BIOS implementation.",
 510         default => 'seabios',
 511     },
 512 };
 513
 514 # what about other qemu settings ?
 515 #cpu => 'string',
 516 #machine => 'string',
 517 #fda => 'file',
 518 #fdb => 'file',
 519 #mtdblock => 'file',
 520 #sd => 'file',
 521 #pflash => 'file',
 522 #snapshot => 'bool',
 523 #bootp => 'file',
 524 ##tftp => 'dir',
 525 ##smb => 'dir',
 526 #kernel => 'file',
 527 #append => 'string',
 528 #initrd => 'file',
 529 ##soundhw => 'string',
 530
 531 while (my ($k, $v) = each %$confdesc) {
 532     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 533 }
 534
 535 my $MAX_IDE_DISKS = 4;
 536 my $MAX_SCSI_DISKS = 14;
 537 my $MAX_VIRTIO_DISKS = 16;
 538 my $MAX_SATA_DISKS = 6;
 539 my $MAX_USB_DEVICES = 5;
 540 my $MAX_NETS = 32;
 541 my $MAX_UNUSED_DISKS = 8;
 542 my $MAX_HOSTPCI_DEVICES = 4;
 543 my $MAX_SERIAL_PORTS = 4;
 544 my $MAX_PARALLEL_PORTS = 3;
 545 my $MAX_NUMA = 8;
 546
 547 my $numa_fmt = {
 548     cpus => {
 549         type => "string",
 550         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 551         description => "CPUs accessing this NUMA node.",
 552         format_description => "id[-id];...",
 553     },
 554     memory => {
 555         type => "number",
 556         description => "Amount of memory this NUMA node provides.",
 557         optional => 1,
 558     },
 559     hostnodes => {
 560         type => "string",
 561         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 562         description => "Host NUMA nodes to use.",
 563         format_description => "id[-id];...",
 564         optional => 1,
 565     },
 566     policy => {
 567         type => 'string',
 568         enum => [qw(preferred bind interleave)],
 569         description => "NUMA allocation policy.",
 570         optional => 1,
 571     },
 572 };
 573 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 574 my $numadesc = {
 575     optional => 1,
 576     type => 'string', format => $numa_fmt,
 577     description => "NUMA topology.",
 578 };
 579 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 580
 581 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 582     $confdesc->{"numa$i"} = $numadesc;
 583 }
 584
 585 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 586                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 587                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 588 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 589
 590 my $net_fmt_bridge_descr = <<__EOD__;
 591 Bridge to attach the network device to. The Proxmox VE standard bridge
 592 is called 'vmbr0'.
 593
 594 If you do not specify a bridge, we create a kvm user (NATed) network
 595 device, which provides DHCP and DNS services. The following addresses
 596 are used:
 597
 598  10.0.2.2   Gateway
 599  10.0.2.3   DNS Server
 600  10.0.2.4   SMB Server
 601
 602 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 603 __EOD__
 604
 605 my $net_fmt = {
 606     macaddr => {
 607         type => 'string',
 608         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 609         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 610         format_description => "XX:XX:XX:XX:XX:XX",
 611         optional => 1,
 612     },
 613     model => {
 614         type => 'string',
 615         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 616         enum => $nic_model_list,
 617         default_key => 1,
 618     },
 619     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 620     bridge => {
 621         type => 'string',
 622         description => $net_fmt_bridge_descr,
 623         format_description => 'bridge',
 624         optional => 1,
 625     },
 626     queues => {
 627         type => 'integer',
 628         minimum => 0, maximum => 16,
 629         description => 'Number of packet queues to be used on the device.',
 630         optional => 1,
 631     },
 632     rate => {
 633         type => 'number',
 634         minimum => 0,
 635         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 636         optional => 1,
 637     },
 638     tag => {
 639         type => 'integer',
 640         minimum => 1, maximum => 4094,
 641         description => 'VLAN tag to apply to packets on this interface.',
 642         optional => 1,
 643     },
 644     trunks => {
 645         type => 'string',
 646         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 647         description => 'VLAN trunks to pass through this interface.',
 648         format_description => 'vlanid[;vlanid...]',
 649         optional => 1,
 650     },
 651     firewall => {
 652         type => 'boolean',
 653         description => 'Whether this interface should be protected by the firewall.',
 654         optional => 1,
 655     },
 656     link_down => {
 657         type => 'boolean',
 658         description => 'Whether this interface should be disconnected (like pulling the plug).',
 659         optional => 1,
 660     },
 661 };
 662
 663 my $netdesc = {
 664     optional => 1,
 665     type => 'string', format => $net_fmt,
 666     description => "Specify network devices.",
 667 };
 668
 669 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 670
 671 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 672     $confdesc->{"net$i"} = $netdesc;
 673 }
 674
 675 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 676 sub verify_volume_id_or_qm_path {
 677     my ($volid, $noerr) = @_;
 678
 679     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 680         return $volid;
 681     }
 682
 683     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 684     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 685     if ($@) {
 686         return undef if $noerr;
 687         die $@;
 688     }
 689     return $volid;
 690 }
 691
 692 my $drivename_hash;
 693
 694 my %drivedesc_base = (
 695     volume => { alias => 'file' },
 696     file => {
 697         type => 'string',
 698         format => 'pve-volume-id-or-qm-path',
 699         default_key => 1,
 700         format_description => 'volume',
 701         description => "The drive's backing volume.",
 702     },
 703     media => {
 704         type => 'string',
 705         enum => [qw(cdrom disk)],
 706         description => "The drive's media type.",
 707         default => 'disk',
 708         optional => 1
 709     },
 710     cyls => {
 711         type => 'integer',
 712         description => "Force the drive's physical geometry to have a specific cylinder count.",
 713         optional => 1
 714     },
 715     heads => {
 716         type => 'integer',
 717         description => "Force the drive's physical geometry to have a specific head count.",
 718         optional => 1
 719     },
 720     secs => {
 721         type => 'integer',
 722         description => "Force the drive's physical geometry to have a specific sector count.",
 723         optional => 1
 724     },
 725     trans => {
 726         type => 'string',
 727         enum => [qw(none lba auto)],
 728         description => "Force disk geometry bios translation mode.",
 729         optional => 1,
 730     },
 731     snapshot => {
 732         type => 'boolean',
 733         description => "Whether the drive should be included when making snapshots.",
 734         optional => 1,
 735     },
 736     cache => {
 737         type => 'string',
 738         enum => [qw(none writethrough writeback unsafe directsync)],
 739         description => "The drive's cache mode",
 740         optional => 1,
 741     },
 742     format => get_standard_option('pve-qm-image-format'),
 743     size => {
 744         type => 'string',
 745         format => 'disk-size',
 746         format_description => 'DiskSize',
 747         description => "Disk size. This is purely informational and has no effect.",
 748         optional => 1,
 749     },
 750     backup => {
 751         type => 'boolean',
 752         description => "Whether the drive should be included when making backups.",
 753         optional => 1,
 754     },
 755     werror => {
 756         type => 'string',
 757         enum => [qw(enospc ignore report stop)],
 758         description => 'Write error action.',
 759         optional => 1,
 760     },
 761     aio => {
 762         type => 'string',
 763         enum => [qw(native threads)],
 764         description => 'AIO type to use.',
 765         optional => 1,
 766     },
 767     discard => {
 768         type => 'string',
 769         enum => [qw(ignore on)],
 770         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 771         optional => 1,
 772     },
 773     detect_zeroes => {
 774         type => 'boolean',
 775         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 776         optional => 1,
 777     },
 778     serial => {
 779         type => 'string',
 780         format => 'urlencoded',
 781         format_description => 'serial',
 782         maxLength => 20*3, # *3 since it's %xx url enoded
 783         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 784         optional => 1,
 785     }
 786 );
 787
 788 my %rerror_fmt = (
 789     rerror => {
 790         type => 'string',
 791         enum => [qw(ignore report stop)],
 792         description => 'Read error action.',
 793         optional => 1,
 794     },
 795 );
 796
 797 my %iothread_fmt = ( iothread => {
 798         type => 'boolean',
 799         description => "Whether to use iothreads for this drive",
 800         optional => 1,
 801 });
 802
 803 my %model_fmt = (
 804     model => {
 805         type => 'string',
 806         format => 'urlencoded',
 807         format_description => 'model',
 808         maxLength => 40*3, # *3 since it's %xx url enoded
 809         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 810         optional => 1,
 811     },
 812 );
 813
 814 my %queues_fmt = (
 815     queues => {
 816         type => 'integer',
 817         description => "Number of queues.",
 818         minimum => 2,
 819         optional => 1
 820     }
 821 );
 822
 823 my $add_throttle_desc = sub {
 824     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 825     my $d = {
 826         type => $type,
 827         format_description => $unit,
 828         description => "Maximum $what in $longunit.",
 829         optional => 1,
 830     };
 831     $d->{minimum} = $minimum if defined($minimum);
 832     $drivedesc_base{$key} = $d;
 833 };
 834 # throughput: (leaky bucket)
 835 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 836 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 837 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 838 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 839 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 840 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 841 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 842 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 843 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 844
 845 # pools: (pool of IO before throttling starts taking effect)
 846 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 847 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 848 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 849 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 850 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 851 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 852
 853 # burst lengths
 854 $add_throttle_desc->('bps_max_length',  'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 855 $add_throttle_desc->('bps_rd_length',   'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 856 $add_throttle_desc->('bps_wr_length',   'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 857 $add_throttle_desc->('iops_max_length', 'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 858 $add_throttle_desc->('iops_rd_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 859 $add_throttle_desc->('iops_wr_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 860
 861 my $ide_fmt = {
 862     %drivedesc_base,
 863     %rerror_fmt,
 864     %model_fmt,
 865 };
 866 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 867
 868 my $idedesc = {
 869     optional => 1,
 870     type => 'string', format => $ide_fmt,
 871     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 872 };
 873 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 874
 875 my $scsi_fmt = {
 876     %drivedesc_base,
 877     %iothread_fmt,
 878     %queues_fmt,
 879 };
 880 my $scsidesc = {
 881     optional => 1,
 882     type => 'string', format => $scsi_fmt,
 883     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 884 };
 885 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 886
 887 my $sata_fmt = {
 888     %drivedesc_base,
 889     %rerror_fmt,
 890 };
 891 my $satadesc = {
 892     optional => 1,
 893     type => 'string', format => $sata_fmt,
 894     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 895 };
 896 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 897
 898 my $virtio_fmt = {
 899     %drivedesc_base,
 900     %iothread_fmt,
 901     %rerror_fmt,
 902 };
 903 my $virtiodesc = {
 904     optional => 1,
 905     type => 'string', format => $virtio_fmt,
 906     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 907 };
 908 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 909
 910 my $alldrive_fmt = {
 911     %drivedesc_base,
 912     %rerror_fmt,
 913     %iothread_fmt,
 914     %model_fmt,
 915     %queues_fmt,
 916 };
 917
 918 my $efidisk_fmt = {
 919     volume => { alias => 'file' },
 920     file => {
 921         type => 'string',
 922         format => 'pve-volume-id-or-qm-path',
 923         default_key => 1,
 924         format_description => 'volume',
 925         description => "The drive's backing volume.",
 926     },
 927     format => get_standard_option('pve-qm-image-format'),
 928     size => {
 929         type => 'string',
 930         format => 'disk-size',
 931         format_description => 'DiskSize',
 932         description => "Disk size. This is purely informational and has no effect.",
 933         optional => 1,
 934     },
 935 };
 936
 937 my $efidisk_desc = {
 938     optional => 1,
 939     type => 'string', format => $efidisk_fmt,
 940     description => "Configure a Disk for storing EFI vars",
 941 };
 942
 943 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 944
 945 my $usb_fmt = {
 946     host => {
 947         default_key => 1,
 948         type => 'string', format => 'pve-qm-usb-device',
 949         format_description => 'HOSTUSBDEVICE|spice',
 950         description => <<EODESCR,
 951 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 952
 953  'bus-port(.port)*' (decimal numbers) or
 954  'vendor_id:product_id' (hexadeciaml numbers) or
 955  'spice'
 956
 957 You can use the 'lsusb -t' command to list existing usb devices.
 958
 959 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 960
 961 The value 'spice' can be used to add a usb redirection devices for spice.
 962 EODESCR
 963     },
 964     usb3 => {
 965         optional => 1,
 966         type => 'boolean',
 967         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 968         default => 0,
 969     },
 970 };
 971
 972 my $usbdesc = {
 973     optional => 1,
 974     type => 'string', format => $usb_fmt,
 975     description => "Configure an USB device (n is 0 to 4).",
 976 };
 977 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 978
 979 # NOTE: the match-groups of this regex are used in parse_hostpci
 980 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 981 my $hostpci_fmt = {
 982     host => {
 983         default_key => 1,
 984         type => 'string',
 985         pattern => qr/$PCIRE(;$PCIRE)*/,
 986         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 987         description => <<EODESCR,
 988 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 989 of PCI virtual functions of the host. HOSTPCIID syntax is:
 990
 991 'bus:dev.func' (hexadecimal numbers)
 992
 993 You can us the 'lspci' command to list existing PCI devices.
 994 EODESCR
 995     },
 996     rombar => {
 997         type => 'boolean',
 998         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
 999         optional => 1,
1000         default => 1,
1001     },
1002     pcie => {
1003         type => 'boolean',
1004         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1005         optional => 1,
1006         default => 0,
1007     },
1008     'x-vga' => {
1009         type => 'boolean',
1010         description =>  "Enable vfio-vga device support.",
1011         optional => 1,
1012         default => 0,
1013     },
1014 };
1015 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1016
1017 my $hostpcidesc = {
1018         optional => 1,
1019         type => 'string', format => 'pve-qm-hostpci',
1020         description => "Map host PCI devices into guest.",
1021         verbose_description =>  <<EODESCR,
1022 Map host PCI devices into guest.
1023
1024 NOTE: This option allows direct access to host hardware. So it is no longer
1025 possible to migrate such machines - use with special care.
1026
1027 CAUTION: Experimental! User reported problems with this option.
1028 EODESCR
1029 };
1030 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1031
1032 my $serialdesc = {
1033         optional => 1,
1034         type => 'string',
1035         pattern => '(/dev/.+|socket)',
1036         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1037         verbose_description =>  <<EODESCR,
1038 Create a serial device inside the VM (n is 0 to 3), and pass through a
1039 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1040 host side (use 'qm terminal' to open a terminal connection).
1041
1042 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1043
1044 CAUTION: Experimental! User reported problems with this option.
1045 EODESCR
1046 };
1047
1048 my $paralleldesc= {
1049         optional => 1,
1050         type => 'string',
1051         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1052         description =>  "Map host parallel devices (n is 0 to 2).",
1053         verbose_description =>  <<EODESCR,
1054 Map host parallel devices (n is 0 to 2).
1055
1056 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1057
1058 CAUTION: Experimental! User reported problems with this option.
1059 EODESCR
1060 };
1061
1062 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1063     $confdesc->{"parallel$i"} = $paralleldesc;
1064 }
1065
1066 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1067     $confdesc->{"serial$i"} = $serialdesc;
1068 }
1069
1070 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1071     $confdesc->{"hostpci$i"} = $hostpcidesc;
1072 }
1073
1074 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1075     $drivename_hash->{"ide$i"} = 1;
1076     $confdesc->{"ide$i"} = $idedesc;
1077 }
1078
1079 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1080     $drivename_hash->{"sata$i"} = 1;
1081     $confdesc->{"sata$i"} = $satadesc;
1082 }
1083
1084 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1085     $drivename_hash->{"scsi$i"} = 1;
1086     $confdesc->{"scsi$i"} = $scsidesc ;
1087 }
1088
1089 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1090     $drivename_hash->{"virtio$i"} = 1;
1091     $confdesc->{"virtio$i"} = $virtiodesc;
1092 }
1093
1094 $drivename_hash->{efidisk0} = 1;
1095 $confdesc->{efidisk0} = $efidisk_desc;
1096
1097 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1098     $confdesc->{"usb$i"} = $usbdesc;
1099 }
1100
1101 my $unuseddesc = {
1102     optional => 1,
1103     type => 'string', format => 'pve-volume-id',
1104     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1105 };
1106
1107 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1108     $confdesc->{"unused$i"} = $unuseddesc;
1109 }
1110
1111 my $kvm_api_version = 0;
1112
1113 sub kvm_version {
1114
1115     return $kvm_api_version if $kvm_api_version;
1116
1117     my $fh = IO::File->new("</dev/kvm") ||
1118         return 0;
1119
1120     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1121         $kvm_api_version = $v;
1122     }
1123
1124     $fh->close();
1125
1126     return  $kvm_api_version;
1127 }
1128
1129 my $kvm_user_version;
1130
1131 sub kvm_user_version {
1132
1133     return $kvm_user_version if $kvm_user_version;
1134
1135     $kvm_user_version = 'unknown';
1136
1137     my $code = sub {
1138         my $line = shift;
1139         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1140             $kvm_user_version = $2;
1141         }
1142     };
1143
1144     eval { run_command("kvm -version", outfunc => $code); };
1145     warn $@ if $@;
1146
1147     return $kvm_user_version;
1148
1149 }
1150
1151 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1152
1153 sub valid_drive_names {
1154     # order is important - used to autoselect boot disk
1155     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1156             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1157             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1158             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1159             'efidisk0');
1160 }
1161
1162 sub is_valid_drivename {
1163     my $dev = shift;
1164
1165     return defined($drivename_hash->{$dev});
1166 }
1167
1168 sub option_exists {
1169     my $key = shift;
1170     return defined($confdesc->{$key});
1171 }
1172
1173 sub nic_models {
1174     return $nic_model_list;
1175 }
1176
1177 sub os_list_description {
1178
1179     return {
1180         other => 'Other',
1181         wxp => 'Windows XP',
1182         w2k => 'Windows 2000',
1183         w2k3 =>, 'Windows 2003',
1184         w2k8 => 'Windows 2008',
1185         wvista => 'Windows Vista',
1186         win7 => 'Windows 7',
1187         win8 => 'Windows 8/2012',
1188         l24 => 'Linux 2.4',
1189         l26 => 'Linux 2.6',
1190     };
1191 }
1192
1193 my $cdrom_path;
1194
1195 sub get_cdrom_path {
1196
1197     return  $cdrom_path if $cdrom_path;
1198
1199     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1200     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1201     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1202 }
1203
1204 sub get_iso_path {
1205     my ($storecfg, $vmid, $cdrom) = @_;
1206
1207     if ($cdrom eq 'cdrom') {
1208         return get_cdrom_path();
1209     } elsif ($cdrom eq 'none') {
1210         return '';
1211     } elsif ($cdrom =~ m|^/|) {
1212         return $cdrom;
1213     } else {
1214         return PVE::Storage::path($storecfg, $cdrom);
1215     }
1216 }
1217
1218 # try to convert old style file names to volume IDs
1219 sub filename_to_volume_id {
1220     my ($vmid, $file, $media) = @_;
1221
1222     if (!($file eq 'none' || $file eq 'cdrom' ||
1223           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1224
1225         return undef if $file =~ m|/|;
1226
1227         if ($media && $media eq 'cdrom') {
1228             $file = "local:iso/$file";
1229         } else {
1230             $file = "local:$vmid/$file";
1231         }
1232     }
1233
1234     return $file;
1235 }
1236
1237 sub verify_media_type {
1238     my ($opt, $vtype, $media) = @_;
1239
1240     return if !$media;
1241
1242     my $etype;
1243     if ($media eq 'disk') {
1244         $etype = 'images';
1245     } elsif ($media eq 'cdrom') {
1246         $etype = 'iso';
1247     } else {
1248         die "internal error";
1249     }
1250
1251     return if ($vtype eq $etype);
1252
1253     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1254 }
1255
1256 sub cleanup_drive_path {
1257     my ($opt, $storecfg, $drive) = @_;
1258
1259     # try to convert filesystem paths to volume IDs
1260
1261     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1262         ($drive->{file} !~ m|^/dev/.+|) &&
1263         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1264         ($drive->{file} !~ m/^\d+$/)) {
1265         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1266         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1267         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1268         verify_media_type($opt, $vtype, $drive->{media});
1269         $drive->{file} = $volid;
1270     }
1271
1272     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1273 }
1274
1275 sub parse_hotplug_features {
1276     my ($data) = @_;
1277
1278     my $res = {};
1279
1280     return $res if $data eq '0';
1281
1282     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1283
1284     foreach my $feature (PVE::Tools::split_list($data)) {
1285         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1286             $res->{$1} = 1;
1287         } else {
1288             die "invalid hotplug feature '$feature'\n";
1289         }
1290     }
1291     return $res;
1292 }
1293
1294 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1295 sub pve_verify_hotplug_features {
1296     my ($value, $noerr) = @_;
1297
1298     return $value if parse_hotplug_features($value);
1299
1300     return undef if $noerr;
1301
1302     die "unable to parse hotplug option\n";
1303 }
1304
1305 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1306 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1307 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1308 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1309 #        [,iothread=on][,serial=serial][,model=model]
1310
1311 sub parse_drive {
1312     my ($key, $data) = @_;
1313
1314     my ($interface, $index);
1315
1316     if ($key =~ m/^([^\d]+)(\d+)$/) {
1317         $interface = $1;
1318         $index = $2;
1319     } else {
1320         return undef;
1321     }
1322
1323     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1324                                      : $confdesc->{$key}->{format};
1325     if (!$desc) {
1326         warn "invalid drive key: $key\n";
1327         return undef;
1328     }
1329     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1330     return undef if !$res;
1331     $res->{interface} = $interface;
1332     $res->{index} = $index;
1333
1334     my $error = 0;
1335     foreach my $opt (qw(bps bps_rd bps_wr)) {
1336         if (my $bps = defined(delete $res->{$opt})) {
1337             if (defined($res->{"m$opt"})) {
1338                 warn "both $opt and m$opt specified\n";
1339                 ++$error;
1340                 next;
1341             }
1342             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1343         }
1344     }
1345
1346     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1347     for my $requirement (
1348         [bps_max_length => 'mbps_max'],
1349         [bps_rd_max_length => 'mbps_rd_max'],
1350         [bps_wr_max_length => 'mbps_wr_max'],
1351         [iops_max_length => 'iops_max'],
1352         [iops_rd_max_length => 'iops_rd_max'],
1353         [iops_wr_max_length => 'iops_wr_max']) {
1354         my ($option, $requires) = @$requirement;
1355         if ($res->{$option} && !$res->{$requires}) {
1356             warn "$option requires $requires\n";
1357             ++$error;
1358         }
1359     }
1360
1361     return undef if $error;
1362
1363     return undef if $res->{mbps_rd} && $res->{mbps};
1364     return undef if $res->{mbps_wr} && $res->{mbps};
1365     return undef if $res->{iops_rd} && $res->{iops};
1366     return undef if $res->{iops_wr} && $res->{iops};
1367
1368     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1369         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1370         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1371         return undef if $res->{interface} eq 'virtio';
1372     }
1373
1374     if (my $size = $res->{size}) {
1375         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1376     }
1377
1378     return $res;
1379 }
1380
1381 sub print_drive {
1382     my ($vmid, $drive) = @_;
1383     my $data = { %$drive };
1384     delete $data->{$_} for qw(index interface);
1385     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1386 }
1387
1388 sub scsi_inquiry {
1389     my($fh, $noerr) = @_;
1390
1391     my $SG_IO = 0x2285;
1392     my $SG_GET_VERSION_NUM = 0x2282;
1393
1394     my $versionbuf = "\x00" x 8;
1395     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1396     if (!$ret) {
1397         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1398         return undef;
1399     }
1400     my $version = unpack("I", $versionbuf);
1401     if ($version < 30000) {
1402         die "scsi generic interface too old\n"  if !$noerr;
1403         return undef;
1404     }
1405
1406     my $buf = "\x00" x 36;
1407     my $sensebuf = "\x00" x 8;
1408     my $cmd = pack("C x3 C x1", 0x12, 36);
1409
1410     # see /usr/include/scsi/sg.h
1411     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1412
1413     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1414                       length($sensebuf), 0, length($buf), $buf,
1415                       $cmd, $sensebuf, 6000);
1416
1417     $ret = ioctl($fh, $SG_IO, $packet);
1418     if (!$ret) {
1419         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1420         return undef;
1421     }
1422
1423     my @res = unpack($sg_io_hdr_t, $packet);
1424     if ($res[17] || $res[18]) {
1425         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1426         return undef;
1427     }
1428
1429     my $res = {};
1430     (my $byte0, my $byte1, $res->{vendor},
1431      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1432
1433     $res->{removable} = $byte1 & 128 ? 1 : 0;
1434     $res->{type} = $byte0 & 31;
1435
1436     return $res;
1437 }
1438
1439 sub path_is_scsi {
1440     my ($path) = @_;
1441
1442     my $fh = IO::File->new("+<$path") || return undef;
1443     my $res = scsi_inquiry($fh, 1);
1444     close($fh);
1445
1446     return $res;
1447 }
1448
1449 sub machine_type_is_q35 {
1450     my ($conf) = @_;
1451
1452     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1453 }
1454
1455 sub print_tabletdevice_full {
1456     my ($conf) = @_;
1457
1458     my $q35 = machine_type_is_q35($conf);
1459
1460     # we use uhci for old VMs because tablet driver was buggy in older qemu
1461     my $usbbus = $q35 ? "ehci" : "uhci";
1462
1463     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1464 }
1465
1466 sub print_drivedevice_full {
1467     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1468
1469     my $device = '';
1470     my $maxdev = 0;
1471
1472     if ($drive->{interface} eq 'virtio') {
1473         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1474         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1475         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1476     } elsif ($drive->{interface} eq 'scsi') {
1477
1478         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1479         my $unit = $drive->{index} % $maxdev;
1480         my $devicetype = 'hd';
1481         my $path = '';
1482         if (drive_is_cdrom($drive)) {
1483             $devicetype = 'cd';
1484         } else {
1485             if ($drive->{file} =~ m|^/|) {
1486                 $path = $drive->{file};
1487                 if (my $info = path_is_scsi($path)) {
1488                     if ($info->{type} == 0) {
1489                         $devicetype = 'block';
1490                     } elsif ($info->{type} == 1) { # tape
1491                         $devicetype = 'generic';
1492                     }
1493                 }
1494             } else {
1495                  $path = PVE::Storage::path($storecfg, $drive->{file});
1496             }
1497
1498             if($path =~ m/^iscsi\:\/\//){
1499                 $devicetype = 'generic';
1500             }
1501         }
1502
1503         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1504             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1505         } else {
1506             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1507         }
1508
1509     } elsif ($drive->{interface} eq 'ide'){
1510         $maxdev = 2;
1511         my $controller = int($drive->{index} / $maxdev);
1512         my $unit = $drive->{index} % $maxdev;
1513         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1514
1515         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1516         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1517             $model = URI::Escape::uri_unescape($model);
1518             $device .= ",model=$model";
1519         }
1520     } elsif ($drive->{interface} eq 'sata'){
1521         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1522         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1523         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1524     } elsif ($drive->{interface} eq 'usb') {
1525         die "implement me";
1526         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1527     } else {
1528         die "unsupported interface type";
1529     }
1530
1531     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1532
1533     return $device;
1534 }
1535
1536 sub get_initiator_name {
1537     my $initiator;
1538
1539     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1540     while (defined(my $line = <$fh>)) {
1541         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1542         $initiator = $1;
1543         last;
1544     }
1545     $fh->close();
1546
1547     return $initiator;
1548 }
1549
1550 sub print_drive_full {
1551     my ($storecfg, $vmid, $drive) = @_;
1552
1553     my $path;
1554     my $volid = $drive->{file};
1555     my $format;
1556
1557     if (drive_is_cdrom($drive)) {
1558         $path = get_iso_path($storecfg, $vmid, $volid);
1559     } else {
1560         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1561         if ($storeid) {
1562             $path = PVE::Storage::path($storecfg, $volid);
1563             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1564             $format = qemu_img_format($scfg, $volname);
1565         } else {
1566             $path = $volid;
1567             $format = "raw";
1568         }
1569    }
1570
1571     my $opts = '';
1572     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1573     foreach my $o (@qemu_drive_options) {
1574         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1575     }
1576     if (my $serial = $drive->{serial}) {
1577         $serial = URI::Escape::uri_unescape($serial);
1578         $opts .= ",serial=$serial";
1579     }
1580
1581     $opts .= ",format=$format" if $format && !$drive->{format};
1582
1583     foreach my $o (qw(bps bps_rd bps_wr)) {
1584         my $v = $drive->{"m$o"};
1585         $opts .= ",$o=" . int($v*1024*1024) if $v;
1586     }
1587
1588     my $cache_direct = 0;
1589
1590     if (my $cache = $drive->{cache}) {
1591         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1592     } elsif (!drive_is_cdrom($drive)) {
1593         $opts .= ",cache=none";
1594         $cache_direct = 1;
1595     }
1596
1597     # aio native works only with O_DIRECT
1598     if (!$drive->{aio}) {
1599         if($cache_direct) {
1600             $opts .= ",aio=native";
1601         } else {
1602             $opts .= ",aio=threads";
1603         }
1604     }
1605
1606     if (!drive_is_cdrom($drive)) {
1607         my $detectzeroes;
1608         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1609             $detectzeroes = 'off';
1610         } elsif ($drive->{discard}) {
1611             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1612         } else {
1613             # This used to be our default with discard not being specified:
1614             $detectzeroes = 'on';
1615         }
1616         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1617     }
1618
1619     my $pathinfo = $path ? "file=$path," : '';
1620
1621     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1622 }
1623
1624 sub print_netdevice_full {
1625     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1626
1627     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1628
1629     my $device = $net->{model};
1630     if ($net->{model} eq 'virtio') {
1631          $device = 'virtio-net-pci';
1632      };
1633
1634     my $pciaddr = print_pci_addr("$netid", $bridges);
1635     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1636     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1637         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1638         my $vectors = $net->{queues} * 2 + 2;
1639         $tmpstr .= ",vectors=$vectors,mq=on";
1640     }
1641     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1642
1643     if ($use_old_bios_files) {
1644         my $romfile;
1645         if ($device eq 'virtio-net-pci') {
1646             $romfile = 'pxe-virtio.rom';
1647         } elsif ($device eq 'e1000') {
1648             $romfile = 'pxe-e1000.rom';
1649         } elsif ($device eq 'ne2k') {
1650             $romfile = 'pxe-ne2k_pci.rom';
1651         } elsif ($device eq 'pcnet') {
1652             $romfile = 'pxe-pcnet.rom';
1653         } elsif ($device eq 'rtl8139') {
1654             $romfile = 'pxe-rtl8139.rom';
1655         }
1656         $tmpstr .= ",romfile=$romfile" if $romfile;
1657     }
1658
1659     return $tmpstr;
1660 }
1661
1662 sub print_netdev_full {
1663     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1664
1665     my $i = '';
1666     if ($netid =~ m/^net(\d+)$/) {
1667         $i = int($1);
1668     }
1669
1670     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1671
1672     my $ifname = "tap${vmid}i$i";
1673
1674     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1675     die "interface name '$ifname' is too long (max 15 character)\n"
1676         if length($ifname) >= 16;
1677
1678     my $vhostparam = '';
1679     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1680
1681     my $vmname = $conf->{name} || "vm$vmid";
1682
1683     my $netdev = "";
1684     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1685
1686     if ($net->{bridge}) {
1687         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1688     } else {
1689         $netdev = "type=user,id=$netid,hostname=$vmname";
1690     }
1691
1692     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1693
1694     return $netdev;
1695 }
1696
1697
1698 sub print_cpu_device {
1699     my ($conf, $id) = @_;
1700
1701     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
1702     my $cpu = $nokvm ? "qemu64" : "kvm64";
1703     if (my $cputype = $conf->{cpu}) {
1704         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1705             or die "Cannot parse cpu description: $cputype\n";
1706         $cpu = $cpuconf->{cputype};
1707     }
1708
1709     my $sockets = 1;
1710     $sockets = $conf->{sockets} if  $conf->{sockets};
1711     my $cores = $conf->{cores} || 1;
1712
1713     my $current_core = ($id - 1) % $cores;
1714     my $current_socket = int(($id - $current_core)/$cores);
1715
1716     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1717 }
1718
1719 sub drive_is_cdrom {
1720     my ($drive) = @_;
1721
1722     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1723
1724 }
1725
1726 sub parse_number_sets {
1727     my ($set) = @_;
1728     my $res = [];
1729     foreach my $part (split(/;/, $set)) {
1730         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1731             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1732             push @$res, [ $1, $2 ];
1733         } else {
1734             die "invalid range: $part\n";
1735         }
1736     }
1737     return $res;
1738 }
1739
1740 sub parse_numa {
1741     my ($data) = @_;
1742
1743     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1744     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1745     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1746     return $res;
1747 }
1748
1749 sub parse_hostpci {
1750     my ($value) = @_;
1751
1752     return undef if !$value;
1753
1754     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1755
1756     my @idlist = split(/;/, $res->{host});
1757     delete $res->{host};
1758     foreach my $id (@idlist) {
1759         if ($id =~ /^$PCIRE$/) {
1760             if (defined($2)) {
1761                 push @{$res->{pciid}}, { id => $1, function => $2 };
1762             } else {
1763                 my $pcidevices = lspci($1);
1764                 $res->{pciid} = $pcidevices->{$1};
1765             }
1766         } else {
1767             # should have been caught by parse_property_string already
1768             die "failed to parse PCI id: $id\n";
1769         }
1770     }
1771     return $res;
1772 }
1773
1774 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1775 sub parse_net {
1776     my ($data) = @_;
1777
1778     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1779     if ($@) {
1780         warn $@;
1781         return undef;
1782     }
1783     if (!defined($res->{macaddr})) {
1784         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1785         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1786     }
1787     return $res;
1788 }
1789
1790 sub print_net {
1791     my $net = shift;
1792
1793     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1794 }
1795
1796 sub add_random_macs {
1797     my ($settings) = @_;
1798
1799     foreach my $opt (keys %$settings) {
1800         next if $opt !~ m/^net(\d+)$/;
1801         my $net = parse_net($settings->{$opt});
1802         next if !$net;
1803         $settings->{$opt} = print_net($net);
1804     }
1805 }
1806
1807 sub vm_is_volid_owner {
1808     my ($storecfg, $vmid, $volid) = @_;
1809
1810     if ($volid !~  m|^/|) {
1811         my ($path, $owner);
1812         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1813         if ($owner && ($owner == $vmid)) {
1814             return 1;
1815         }
1816     }
1817
1818     return undef;
1819 }
1820
1821 sub split_flagged_list {
1822     my $text = shift || '';
1823     $text =~ s/[,;]/ /g;
1824     $text =~ s/^\s+//;
1825     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1826 }
1827
1828 sub join_flagged_list {
1829     my ($how, $lst) = @_;
1830     join $how, map { $lst->{$_} . $_ } keys %$lst;
1831 }
1832
1833 sub vmconfig_delete_pending_option {
1834     my ($conf, $key, $force) = @_;
1835
1836     delete $conf->{pending}->{$key};
1837     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1838     $pending_delete_hash->{$key} = $force ? '!' : '';
1839     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1840 }
1841
1842 sub vmconfig_undelete_pending_option {
1843     my ($conf, $key) = @_;
1844
1845     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1846     delete $pending_delete_hash->{$key};
1847
1848     if (%$pending_delete_hash) {
1849         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1850     } else {
1851         delete $conf->{pending}->{delete};
1852     }
1853 }
1854
1855 sub vmconfig_register_unused_drive {
1856     my ($storecfg, $vmid, $conf, $drive) = @_;
1857
1858     if (!drive_is_cdrom($drive)) {
1859         my $volid = $drive->{file};
1860         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1861             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1862         }
1863     }
1864 }
1865
1866 sub vmconfig_cleanup_pending {
1867     my ($conf) = @_;
1868
1869     # remove pending changes when nothing changed
1870     my $changes;
1871     foreach my $opt (keys %{$conf->{pending}}) {
1872         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1873             $changes = 1;
1874             delete $conf->{pending}->{$opt};
1875         }
1876     }
1877
1878     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1879     my $pending_delete_hash = {};
1880     while (my ($opt, $force) = each %$current_delete_hash) {
1881         if (defined($conf->{$opt})) {
1882             $pending_delete_hash->{$opt} = $force;
1883         } else {
1884             $changes = 1;
1885         }
1886     }
1887
1888     if (%$pending_delete_hash) {
1889         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1890     } else {
1891         delete $conf->{pending}->{delete};
1892     }
1893
1894     return $changes;
1895 }
1896
1897 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1898 my $smbios1_fmt = {
1899     uuid => {
1900         type => 'string',
1901         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1902         format_description => 'UUID',
1903         description => "Set SMBIOS1 UUID.",
1904         optional => 1,
1905     },
1906     version => {
1907         type => 'string',
1908         pattern => '\S+',
1909         format_description => 'string',
1910         description => "Set SMBIOS1 version.",
1911         optional => 1,
1912     },
1913     serial => {
1914         type => 'string',
1915         pattern => '\S+',
1916         format_description => 'string',
1917         description => "Set SMBIOS1 serial number.",
1918         optional => 1,
1919     },
1920     manufacturer => {
1921         type => 'string',
1922         pattern => '\S+',
1923         format_description => 'string',
1924         description => "Set SMBIOS1 manufacturer.",
1925         optional => 1,
1926     },
1927     product => {
1928         type => 'string',
1929         pattern => '\S+',
1930         format_description => 'string',
1931         description => "Set SMBIOS1 product ID.",
1932         optional => 1,
1933     },
1934     sku => {
1935         type => 'string',
1936         pattern => '\S+',
1937         format_description => 'string',
1938         description => "Set SMBIOS1 SKU string.",
1939         optional => 1,
1940     },
1941     family => {
1942         type => 'string',
1943         pattern => '\S+',
1944         format_description => 'string',
1945         description => "Set SMBIOS1 family string.",
1946         optional => 1,
1947     },
1948 };
1949
1950 sub parse_smbios1 {
1951     my ($data) = @_;
1952
1953     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1954     warn $@ if $@;
1955     return $res;
1956 }
1957
1958 sub print_smbios1 {
1959     my ($smbios1) = @_;
1960     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1961 }
1962
1963 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1964
1965 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1966 sub verify_bootdisk {
1967     my ($value, $noerr) = @_;
1968
1969     return $value if is_valid_drivename($value);
1970
1971     return undef if $noerr;
1972
1973     die "invalid boot disk '$value'\n";
1974 }
1975
1976 sub parse_watchdog {
1977     my ($value) = @_;
1978
1979     return undef if !$value;
1980
1981     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1982     warn $@ if $@;
1983     return $res;
1984 }
1985
1986 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1987 sub verify_usb_device {
1988     my ($value, $noerr) = @_;
1989
1990     return $value if parse_usb_device($value);
1991
1992     return undef if $noerr;
1993
1994     die "unable to parse usb device\n";
1995 }
1996
1997 # add JSON properties for create and set function
1998 sub json_config_properties {
1999     my $prop = shift;
2000
2001     foreach my $opt (keys %$confdesc) {
2002         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2003         $prop->{$opt} = $confdesc->{$opt};
2004     }
2005
2006     return $prop;
2007 }
2008
2009 sub check_type {
2010     my ($key, $value) = @_;
2011
2012     die "unknown setting '$key'\n" if !$confdesc->{$key};
2013
2014     my $type = $confdesc->{$key}->{type};
2015
2016     if (!defined($value)) {
2017         die "got undefined value\n";
2018     }
2019
2020     if ($value =~ m/[\n\r]/) {
2021         die "property contains a line feed\n";
2022     }
2023
2024     if ($type eq 'boolean') {
2025         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2026         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2027         die "type check ('boolean') failed - got '$value'\n";
2028     } elsif ($type eq 'integer') {
2029         return int($1) if $value =~ m/^(\d+)$/;
2030         die "type check ('integer') failed - got '$value'\n";
2031     } elsif ($type eq 'number') {
2032         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2033         die "type check ('number') failed - got '$value'\n";
2034     } elsif ($type eq 'string') {
2035         if (my $fmt = $confdesc->{$key}->{format}) {
2036             PVE::JSONSchema::check_format($fmt, $value);
2037             return $value;
2038         }
2039         $value =~ s/^\"(.*)\"$/$1/;
2040         return $value;
2041     } else {
2042         die "internal error"
2043     }
2044 }
2045
2046 sub check_iommu_support{
2047     #fixme : need to check IOMMU support
2048     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2049
2050     my $iommu=1;
2051     return $iommu;
2052
2053 }
2054
2055 sub touch_config {
2056     my ($vmid) = @_;
2057
2058     my $conf = PVE::QemuConfig->config_file($vmid);
2059     utime undef, undef, $conf;
2060 }
2061
2062 sub destroy_vm {
2063     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2064
2065     my $conffile = PVE::QemuConfig->config_file($vmid);
2066
2067     my $conf = PVE::QemuConfig->load_config($vmid);
2068
2069     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2070
2071     # only remove disks owned by this VM
2072     foreach_drive($conf, sub {
2073         my ($ds, $drive) = @_;
2074
2075         return if drive_is_cdrom($drive);
2076
2077         my $volid = $drive->{file};
2078
2079         return if !$volid || $volid =~ m|^/|;
2080
2081         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2082         return if !$path || !$owner || ($owner != $vmid);
2083
2084         PVE::Storage::vdisk_free($storecfg, $volid);
2085     });
2086
2087     if ($keep_empty_config) {
2088         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2089     } else {
2090         unlink $conffile;
2091     }
2092
2093     # also remove unused disk
2094     eval {
2095         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2096
2097         eval {
2098             PVE::Storage::foreach_volid($dl, sub {
2099                 my ($volid, $sid, $volname, $d) = @_;
2100                 PVE::Storage::vdisk_free($storecfg, $volid);
2101             });
2102         };
2103         warn $@ if $@;
2104
2105     };
2106     warn $@ if $@;
2107 }
2108
2109 sub parse_vm_config {
2110     my ($filename, $raw) = @_;
2111
2112     return undef if !defined($raw);
2113
2114     my $res = {
2115         digest => Digest::SHA::sha1_hex($raw),
2116         snapshots => {},
2117         pending => {},
2118     };
2119
2120     $filename =~ m|/qemu-server/(\d+)\.conf$|
2121         || die "got strange filename '$filename'";
2122
2123     my $vmid = $1;
2124
2125     my $conf = $res;
2126     my $descr;
2127     my $section = '';
2128
2129     my @lines = split(/\n/, $raw);
2130     foreach my $line (@lines) {
2131         next if $line =~ m/^\s*$/;
2132
2133         if ($line =~ m/^\[PENDING\]\s*$/i) {
2134             $section = 'pending';
2135             if (defined($descr)) {
2136                 $descr =~ s/\s+$//;
2137                 $conf->{description} = $descr;
2138             }
2139             $descr = undef;
2140             $conf = $res->{$section} = {};
2141             next;
2142
2143         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2144             $section = $1;
2145             if (defined($descr)) {
2146                 $descr =~ s/\s+$//;
2147                 $conf->{description} = $descr;
2148             }
2149             $descr = undef;
2150             $conf = $res->{snapshots}->{$section} = {};
2151             next;
2152         }
2153
2154         if ($line =~ m/^\#(.*)\s*$/) {
2155             $descr = '' if !defined($descr);
2156             $descr .= PVE::Tools::decode_text($1) . "\n";
2157             next;
2158         }
2159
2160         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2161             $descr = '' if !defined($descr);
2162             $descr .= PVE::Tools::decode_text($2);
2163         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2164             $conf->{snapstate} = $1;
2165         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2166             my $key = $1;
2167             my $value = $2;
2168             $conf->{$key} = $value;
2169         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2170             my $value = $1;
2171             if ($section eq 'pending') {
2172                 $conf->{delete} = $value; # we parse this later
2173             } else {
2174                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2175             }
2176         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2177             my $key = $1;
2178             my $value = $2;
2179             eval { $value = check_type($key, $value); };
2180             if ($@) {
2181                 warn "vm $vmid - unable to parse value of '$key' - $@";
2182             } else {
2183                 $key = 'ide2' if $key eq 'cdrom';
2184                 my $fmt = $confdesc->{$key}->{format};
2185                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2186                     my $v = parse_drive($key, $value);
2187                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2188                         $v->{file} = $volid;
2189                         $value = print_drive($vmid, $v);
2190                     } else {
2191                         warn "vm $vmid - unable to parse value of '$key'\n";
2192                         next;
2193                     }
2194                 }
2195
2196                 $conf->{$key} = $value;
2197             }
2198         }
2199     }
2200
2201     if (defined($descr)) {
2202         $descr =~ s/\s+$//;
2203         $conf->{description} = $descr;
2204     }
2205     delete $res->{snapstate}; # just to be sure
2206
2207     return $res;
2208 }
2209
2210 sub write_vm_config {
2211     my ($filename, $conf) = @_;
2212
2213     delete $conf->{snapstate}; # just to be sure
2214
2215     if ($conf->{cdrom}) {
2216         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2217         $conf->{ide2} = $conf->{cdrom};
2218         delete $conf->{cdrom};
2219     }
2220
2221     # we do not use 'smp' any longer
2222     if ($conf->{sockets}) {
2223         delete $conf->{smp};
2224     } elsif ($conf->{smp}) {
2225         $conf->{sockets} = $conf->{smp};
2226         delete $conf->{cores};
2227         delete $conf->{smp};
2228     }
2229
2230     my $used_volids = {};
2231
2232     my $cleanup_config = sub {
2233         my ($cref, $pending, $snapname) = @_;
2234
2235         foreach my $key (keys %$cref) {
2236             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2237                 $key eq 'snapstate' || $key eq 'pending';
2238             my $value = $cref->{$key};
2239             if ($key eq 'delete') {
2240                 die "propertry 'delete' is only allowed in [PENDING]\n"
2241                     if !$pending;
2242                 # fixme: check syntax?
2243                 next;
2244             }
2245             eval { $value = check_type($key, $value); };
2246             die "unable to parse value of '$key' - $@" if $@;
2247
2248             $cref->{$key} = $value;
2249
2250             if (!$snapname && is_valid_drivename($key)) {
2251                 my $drive = parse_drive($key, $value);
2252                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2253             }
2254         }
2255     };
2256
2257     &$cleanup_config($conf);
2258
2259     &$cleanup_config($conf->{pending}, 1);
2260
2261     foreach my $snapname (keys %{$conf->{snapshots}}) {
2262         die "internal error" if $snapname eq 'pending';
2263         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2264     }
2265
2266     # remove 'unusedX' settings if we re-add a volume
2267     foreach my $key (keys %$conf) {
2268         my $value = $conf->{$key};
2269         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2270             delete $conf->{$key};
2271         }
2272     }
2273
2274     my $generate_raw_config = sub {
2275         my ($conf, $pending) = @_;
2276
2277         my $raw = '';
2278
2279         # add description as comment to top of file
2280         if (defined(my $descr = $conf->{description})) {
2281             if ($descr) {
2282                 foreach my $cl (split(/\n/, $descr)) {
2283                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2284                 }
2285             } else {
2286                 $raw .= "#\n" if $pending;
2287             }
2288         }
2289
2290         foreach my $key (sort keys %$conf) {
2291             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2292             $raw .= "$key: $conf->{$key}\n";
2293         }
2294         return $raw;
2295     };
2296
2297     my $raw = &$generate_raw_config($conf);
2298
2299     if (scalar(keys %{$conf->{pending}})){
2300         $raw .= "\n[PENDING]\n";
2301         $raw .= &$generate_raw_config($conf->{pending}, 1);
2302     }
2303
2304     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2305         $raw .= "\n[$snapname]\n";
2306         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2307     }
2308
2309     return $raw;
2310 }
2311
2312 sub load_defaults {
2313
2314     my $res = {};
2315
2316     # we use static defaults from our JSON schema configuration
2317     foreach my $key (keys %$confdesc) {
2318         if (defined(my $default = $confdesc->{$key}->{default})) {
2319             $res->{$key} = $default;
2320         }
2321     }
2322
2323     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2324     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2325
2326     return $res;
2327 }
2328
2329 sub config_list {
2330     my $vmlist = PVE::Cluster::get_vmlist();
2331     my $res = {};
2332     return $res if !$vmlist || !$vmlist->{ids};
2333     my $ids = $vmlist->{ids};
2334
2335     foreach my $vmid (keys %$ids) {
2336         my $d = $ids->{$vmid};
2337         next if !$d->{node} || $d->{node} ne $nodename;
2338         next if !$d->{type} || $d->{type} ne 'qemu';
2339         $res->{$vmid}->{exists} = 1;
2340     }
2341     return $res;
2342 }
2343
2344 # test if VM uses local resources (to prevent migration)
2345 sub check_local_resources {
2346     my ($conf, $noerr) = @_;
2347
2348     my $loc_res = 0;
2349
2350     $loc_res = 1 if $conf->{hostusb}; # old syntax
2351     $loc_res = 1 if $conf->{hostpci}; # old syntax
2352
2353     foreach my $k (keys %$conf) {
2354         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2355         # sockets are safe: they will recreated be on the target side post-migrate
2356         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2357         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2358     }
2359
2360     die "VM uses local resources\n" if $loc_res && !$noerr;
2361
2362     return $loc_res;
2363 }
2364
2365 # check if used storages are available on all nodes (use by migrate)
2366 sub check_storage_availability {
2367     my ($storecfg, $conf, $node) = @_;
2368
2369     foreach_drive($conf, sub {
2370         my ($ds, $drive) = @_;
2371
2372         my $volid = $drive->{file};
2373         return if !$volid;
2374
2375         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2376         return if !$sid;
2377
2378         # check if storage is available on both nodes
2379         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2380         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2381    });
2382 }
2383
2384 # list nodes where all VM images are available (used by has_feature API)
2385 sub shared_nodes {
2386     my ($conf, $storecfg) = @_;
2387
2388     my $nodelist = PVE::Cluster::get_nodelist();
2389     my $nodehash = { map { $_ => 1 } @$nodelist };
2390     my $nodename = PVE::INotify::nodename();
2391
2392     foreach_drive($conf, sub {
2393         my ($ds, $drive) = @_;
2394
2395         my $volid = $drive->{file};
2396         return if !$volid;
2397
2398         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2399         if ($storeid) {
2400             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2401             if ($scfg->{disable}) {
2402                 $nodehash = {};
2403             } elsif (my $avail = $scfg->{nodes}) {
2404                 foreach my $node (keys %$nodehash) {
2405                     delete $nodehash->{$node} if !$avail->{$node};
2406                 }
2407             } elsif (!$scfg->{shared}) {
2408                 foreach my $node (keys %$nodehash) {
2409                     delete $nodehash->{$node} if $node ne $nodename
2410                 }
2411             }
2412         }
2413     });
2414
2415     return $nodehash
2416 }
2417
2418 sub check_cmdline {
2419     my ($pidfile, $pid) = @_;
2420
2421     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2422     if (defined($fh)) {
2423         my $line = <$fh>;
2424         $fh->close;
2425         return undef if !$line;
2426         my @param = split(/\0/, $line);
2427
2428         my $cmd = $param[0];
2429         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2430
2431         for (my $i = 0; $i < scalar (@param); $i++) {
2432             my $p = $param[$i];
2433             next if !$p;
2434             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2435                 my $p = $param[$i+1];
2436                 return 1 if $p && ($p eq $pidfile);
2437                 return undef;
2438             }
2439         }
2440     }
2441     return undef;
2442 }
2443
2444 sub check_running {
2445     my ($vmid, $nocheck, $node) = @_;
2446
2447     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2448
2449     die "unable to find configuration file for VM $vmid - no such machine\n"
2450         if !$nocheck && ! -f $filename;
2451
2452     my $pidfile = pidfile_name($vmid);
2453
2454     if (my $fd = IO::File->new("<$pidfile")) {
2455         my $st = stat($fd);
2456         my $line = <$fd>;
2457         close($fd);
2458
2459         my $mtime = $st->mtime;
2460         if ($mtime > time()) {
2461             warn "file '$filename' modified in future\n";
2462         }
2463
2464         if ($line =~ m/^(\d+)$/) {
2465             my $pid = $1;
2466             if (check_cmdline($pidfile, $pid)) {
2467                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2468                     return $pid;
2469                 }
2470             }
2471         }
2472     }
2473
2474     return undef;
2475 }
2476
2477 sub vzlist {
2478
2479     my $vzlist = config_list();
2480
2481     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2482
2483     while (defined(my $de = $fd->read)) {
2484         next if $de !~ m/^(\d+)\.pid$/;
2485         my $vmid = $1;
2486         next if !defined($vzlist->{$vmid});
2487         if (my $pid = check_running($vmid)) {
2488             $vzlist->{$vmid}->{pid} = $pid;
2489         }
2490     }
2491
2492     return $vzlist;
2493 }
2494
2495 sub disksize {
2496     my ($storecfg, $conf) = @_;
2497
2498     my $bootdisk = $conf->{bootdisk};
2499     return undef if !$bootdisk;
2500     return undef if !is_valid_drivename($bootdisk);
2501
2502     return undef if !$conf->{$bootdisk};
2503
2504     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2505     return undef if !defined($drive);
2506
2507     return undef if drive_is_cdrom($drive);
2508
2509     my $volid = $drive->{file};
2510     return undef if !$volid;
2511
2512     return $drive->{size};
2513 }
2514
2515 my $last_proc_pid_stat;
2516
2517 # get VM status information
2518 # This must be fast and should not block ($full == false)
2519 # We only query KVM using QMP if $full == true (this can be slow)
2520 sub vmstatus {
2521     my ($opt_vmid, $full) = @_;
2522
2523     my $res = {};
2524
2525     my $storecfg = PVE::Storage::config();
2526
2527     my $list = vzlist();
2528     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2529
2530     my $cpucount = $cpuinfo->{cpus} || 1;
2531
2532     foreach my $vmid (keys %$list) {
2533         next if $opt_vmid && ($vmid ne $opt_vmid);
2534
2535         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2536         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2537
2538         my $d = {};
2539         $d->{pid} = $list->{$vmid}->{pid};
2540
2541         # fixme: better status?
2542         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2543
2544         my $size = disksize($storecfg, $conf);
2545         if (defined($size)) {
2546             $d->{disk} = 0; # no info available
2547             $d->{maxdisk} = $size;
2548         } else {
2549             $d->{disk} = 0;
2550             $d->{maxdisk} = 0;
2551         }
2552
2553         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2554         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2555         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2556
2557         $d->{name} = $conf->{name} || "VM $vmid";
2558         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2559
2560         if ($conf->{balloon}) {
2561             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2562             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2563         }
2564
2565         $d->{uptime} = 0;
2566         $d->{cpu} = 0;
2567         $d->{mem} = 0;
2568
2569         $d->{netout} = 0;
2570         $d->{netin} = 0;
2571
2572         $d->{diskread} = 0;
2573         $d->{diskwrite} = 0;
2574
2575         $d->{template} = PVE::QemuConfig->is_template($conf);
2576
2577         $res->{$vmid} = $d;
2578     }
2579
2580     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2581     foreach my $dev (keys %$netdev) {
2582         next if $dev !~ m/^tap([1-9]\d*)i/;
2583         my $vmid = $1;
2584         my $d = $res->{$vmid};
2585         next if !$d;
2586
2587         $d->{netout} += $netdev->{$dev}->{receive};
2588         $d->{netin} += $netdev->{$dev}->{transmit};
2589
2590         if ($full) {
2591             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2592             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2593         }
2594
2595     }
2596
2597     my $ctime = gettimeofday;
2598
2599     foreach my $vmid (keys %$list) {
2600
2601         my $d = $res->{$vmid};
2602         my $pid = $d->{pid};
2603         next if !$pid;
2604
2605         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2606         next if !$pstat; # not running
2607
2608         my $used = $pstat->{utime} + $pstat->{stime};
2609
2610         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2611
2612         if ($pstat->{vsize}) {
2613             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2614         }
2615
2616         my $old = $last_proc_pid_stat->{$pid};
2617         if (!$old) {
2618             $last_proc_pid_stat->{$pid} = {
2619                 time => $ctime,
2620                 used => $used,
2621                 cpu => 0,
2622             };
2623             next;
2624         }
2625
2626         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2627
2628         if ($dtime > 1000) {
2629             my $dutime = $used -  $old->{used};
2630
2631             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2632             $last_proc_pid_stat->{$pid} = {
2633                 time => $ctime,
2634                 used => $used,
2635                 cpu => $d->{cpu},
2636             };
2637         } else {
2638             $d->{cpu} = $old->{cpu};
2639         }
2640     }
2641
2642     return $res if !$full;
2643
2644     my $qmpclient = PVE::QMPClient->new();
2645
2646     my $ballooncb = sub {
2647         my ($vmid, $resp) = @_;
2648
2649         my $info = $resp->{'return'};
2650         return if !$info->{max_mem};
2651
2652         my $d = $res->{$vmid};
2653
2654         # use memory assigned to VM
2655         $d->{maxmem} = $info->{max_mem};
2656         $d->{balloon} = $info->{actual};
2657
2658         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2659             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2660             $d->{freemem} = $info->{free_mem};
2661         }
2662
2663         $d->{ballooninfo} = $info;
2664     };
2665
2666     my $blockstatscb = sub {
2667         my ($vmid, $resp) = @_;
2668         my $data = $resp->{'return'} || [];
2669         my $totalrdbytes = 0;
2670         my $totalwrbytes = 0;
2671
2672         for my $blockstat (@$data) {
2673             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2674             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2675
2676             $blockstat->{device} =~ s/drive-//;
2677             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2678         }
2679         $res->{$vmid}->{diskread} = $totalrdbytes;
2680         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2681     };
2682
2683     my $statuscb = sub {
2684         my ($vmid, $resp) = @_;
2685
2686         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2687         # this fails if ballon driver is not loaded, so this must be
2688         # the last commnand (following command are aborted if this fails).
2689         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2690
2691         my $status = 'unknown';
2692         if (!defined($status = $resp->{'return'}->{status})) {
2693             warn "unable to get VM status\n";
2694             return;
2695         }
2696
2697         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2698     };
2699
2700     foreach my $vmid (keys %$list) {
2701         next if $opt_vmid && ($vmid ne $opt_vmid);
2702         next if !$res->{$vmid}->{pid}; # not running
2703         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2704     }
2705
2706     $qmpclient->queue_execute(undef, 1);
2707
2708     foreach my $vmid (keys %$list) {
2709         next if $opt_vmid && ($vmid ne $opt_vmid);
2710         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2711     }
2712
2713     return $res;
2714 }
2715
2716 sub foreach_drive {
2717     my ($conf, $func, @param) = @_;
2718
2719     foreach my $ds (valid_drive_names()) {
2720         next if !defined($conf->{$ds});
2721
2722         my $drive = parse_drive($ds, $conf->{$ds});
2723         next if !$drive;
2724
2725         &$func($ds, $drive, @param);
2726     }
2727 }
2728
2729 sub foreach_volid {
2730     my ($conf, $func, @param) = @_;
2731
2732     my $volhash = {};
2733
2734     my $test_volid = sub {
2735         my ($volid, $is_cdrom) = @_;
2736
2737         return if !$volid;
2738
2739         $volhash->{$volid} = $is_cdrom || 0;
2740     };
2741
2742     foreach_drive($conf, sub {
2743         my ($ds, $drive) = @_;
2744         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2745     });
2746
2747     foreach my $snapname (keys %{$conf->{snapshots}}) {
2748         my $snap = $conf->{snapshots}->{$snapname};
2749         &$test_volid($snap->{vmstate}, 0);
2750         foreach_drive($snap, sub {
2751             my ($ds, $drive) = @_;
2752             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2753         });
2754     }
2755
2756     foreach my $volid (keys %$volhash) {
2757         &$func($volid, $volhash->{$volid}, @param);
2758     }
2759 }
2760
2761 sub vga_conf_has_spice {
2762     my ($vga) = @_;
2763
2764     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2765
2766     return $1 || 1;
2767 }
2768
2769 sub config_to_command {
2770     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2771
2772     my $cmd = [];
2773     my $globalFlags = [];
2774     my $machineFlags = [];
2775     my $rtcFlags = [];
2776     my $cpuFlags = [];
2777     my $devices = [];
2778     my $pciaddr = '';
2779     my $bridges = {};
2780     my $kvmver = kvm_user_version();
2781     my $vernum = 0; # unknown
2782     my $ostype = $conf->{ostype};
2783     my $winversion = windows_version($ostype);
2784
2785     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2786         $vernum = $1*1000000+$2*1000;
2787     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2788         $vernum = $1*1000000+$2*1000+$3;
2789     }
2790
2791     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2792
2793     my $have_ovz = -f '/proc/vz/vestat';
2794
2795     my $q35 = machine_type_is_q35($conf);
2796     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2797     my $machine_type = $forcemachine || $conf->{machine};
2798     my $use_old_bios_files = undef;
2799     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2800
2801     my $cpuunits = defined($conf->{cpuunits}) ?
2802             $conf->{cpuunits} : $defaults->{cpuunits};
2803
2804     push @$cmd, '/usr/bin/kvm';
2805
2806     push @$cmd, '-id', $vmid;
2807
2808     my $use_virtio = 0;
2809
2810     my $qmpsocket = qmp_socket($vmid);
2811     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2812     push @$cmd, '-mon', "chardev=qmp,mode=control";
2813
2814
2815     push @$cmd, '-pidfile' , pidfile_name($vmid);
2816
2817     push @$cmd, '-daemonize';
2818
2819     if ($conf->{smbios1}) {
2820         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2821     }
2822
2823     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2824         my $ovmfbase;
2825
2826         # prefer the OVMF_CODE variant
2827         if (-f $OVMF_CODE) {
2828             $ovmfbase = $OVMF_CODE;
2829         } elsif (-f $OVMF_IMG) {
2830             $ovmfbase = $OVMF_IMG;
2831         }
2832
2833         die "no uefi base img found\n" if !$ovmfbase;
2834         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmfbase";
2835
2836         if (defined($conf->{efidisk0}) && ($ovmfbase eq $OVMF_CODE)) {
2837             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $conf->{efidisk0});
2838             my $format = $d->{format} // 'raw';
2839             my $path;
2840             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2841             if ($storeid) {
2842                 $path = PVE::Storage::path($storecfg, $d->{file});
2843                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2844                 $format = qemu_img_format($scfg, $volname);
2845             } else {
2846                 $path = $d->{file};
2847                 $format = "raw";
2848             }
2849             push @$cmd, '-drive', "if=pflash,unit=1,id=drive-efidisk0,format=$format,file=$path";
2850         } elsif ($ovmfbase eq $OVMF_CODE) {
2851             warn "using uefi without permanent efivars disk\n";
2852             my $ovmfvar_dst = "/tmp/$vmid-ovmf.fd";
2853             PVE::Tools::file_copy($OVMF_VARS, $ovmfvar_dst, 256*1024);
2854             push @$cmd, '-drive', "if=pflash,unit=1,format=raw,file=$ovmfvar_dst";
2855         } else {
2856             # if the base img is not OVMF_CODE, we do not have to bother
2857             # to create/use a vars image, since it will not be used anyway
2858             # this can only happen if someone manually deletes the OVMF_CODE image
2859             # or has an old pve-qemu-kvm version installed.
2860             # both should not happen, but we ignore it here
2861         }
2862     }
2863
2864
2865     # add usb controllers
2866     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2867     push @$devices, @usbcontrollers if @usbcontrollers;
2868     my $vga = $conf->{vga};
2869
2870     my $qxlnum = vga_conf_has_spice($vga);
2871     $vga = 'qxl' if $qxlnum;
2872
2873     if (!$vga) {
2874         $vga = $winversion >= 6 ? 'std' : 'cirrus';
2875     }
2876
2877     # enable absolute mouse coordinates (needed by vnc)
2878     my $tablet;
2879     if (defined($conf->{tablet})) {
2880         $tablet = $conf->{tablet};
2881     } else {
2882         $tablet = $defaults->{tablet};
2883         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2884         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2885     }
2886
2887     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2888
2889     my $kvm_off = 0;
2890     my $gpu_passthrough;
2891
2892     # host pci devices
2893     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2894         my $d = parse_hostpci($conf->{"hostpci$i"});
2895         next if !$d;
2896
2897         my $pcie = $d->{pcie};
2898         if($pcie){
2899             die "q35 machine model is not enabled" if !$q35;
2900             $pciaddr = print_pcie_addr("hostpci$i");
2901         }else{
2902             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2903         }
2904
2905         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2906         my $xvga = '';
2907         if ($d->{'x-vga'}) {
2908             $xvga = ',x-vga=on';
2909             $kvm_off = 1;
2910             $vga = 'none';
2911             $gpu_passthrough = 1;
2912
2913             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2914                 $xvga = "";
2915             }
2916         }
2917         my $pcidevices = $d->{pciid};
2918         my $multifunction = 1 if @$pcidevices > 1;
2919
2920         my $j=0;
2921         foreach my $pcidevice (@$pcidevices) {
2922
2923             my $id = "hostpci$i";
2924             $id .= ".$j" if $multifunction;
2925             my $addr = $pciaddr;
2926             $addr .= ".$j" if $multifunction;
2927             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2928
2929             if($j == 0){
2930                 $devicestr .= "$rombar$xvga";
2931                 $devicestr .= ",multifunction=on" if $multifunction;
2932             }
2933
2934             push @$devices, '-device', $devicestr;
2935             $j++;
2936         }
2937     }
2938
2939     # usb devices
2940     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
2941     push @$devices, @usbdevices if @usbdevices;
2942     # serial devices
2943     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2944         if (my $path = $conf->{"serial$i"}) {
2945             if ($path eq 'socket') {
2946                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2947                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2948                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2949             } else {
2950                 die "no such serial device\n" if ! -c $path;
2951                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2952                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2953             }
2954         }
2955     }
2956
2957     # parallel devices
2958     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2959         if (my $path = $conf->{"parallel$i"}) {
2960             die "no such parallel device\n" if ! -c $path;
2961             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2962             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2963             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2964         }
2965     }
2966
2967     my $vmname = $conf->{name} || "vm$vmid";
2968
2969     push @$cmd, '-name', $vmname;
2970
2971     my $sockets = 1;
2972     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2973     $sockets = $conf->{sockets} if  $conf->{sockets};
2974
2975     my $cores = $conf->{cores} || 1;
2976
2977     my $maxcpus = $sockets * $cores;
2978
2979     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2980
2981     my $allowed_vcpus = $cpuinfo->{cpus};
2982
2983     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2984         if ($allowed_vcpus < $maxcpus);
2985
2986     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
2987
2988         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2989         for (my $i = 2; $i <= $vcpus; $i++)  {
2990             my $cpustr = print_cpu_device($conf,$i);
2991             push @$cmd, '-device', $cpustr;
2992         }
2993
2994     } else {
2995
2996         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
2997     }
2998     push @$cmd, '-nodefaults';
2999
3000     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3001
3002     my $bootindex_hash = {};
3003     my $i = 1;
3004     foreach my $o (split(//, $bootorder)) {
3005         $bootindex_hash->{$o} = $i*100;
3006         $i++;
3007     }
3008
3009     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3010
3011     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3012
3013     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3014
3015     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3016
3017     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3018         my $socket = vnc_socket($vmid);
3019         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3020     } else {
3021         push @$cmd, '-nographic';
3022     }
3023
3024     # time drift fix
3025     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3026
3027     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3028     my $useLocaltime = $conf->{localtime};
3029
3030     if ($winversion >= 5) { # windows
3031         $useLocaltime = 1 if !defined($conf->{localtime});
3032
3033         # use time drift fix when acpi is enabled
3034         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3035             $tdf = 1 if !defined($conf->{tdf});
3036         }
3037     }
3038
3039     if ($winversion >= 6) {
3040         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3041         push @$cmd, '-no-hpet';
3042     }
3043
3044     push @$rtcFlags, 'driftfix=slew' if $tdf;
3045
3046     if ($nokvm) {
3047         push @$machineFlags, 'accel=tcg';
3048     } else {
3049         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3050     }
3051
3052     if ($machine_type) {
3053         push @$machineFlags, "type=${machine_type}";
3054     }
3055
3056     if ($conf->{startdate}) {
3057         push @$rtcFlags, "base=$conf->{startdate}";
3058     } elsif ($useLocaltime) {
3059         push @$rtcFlags, 'base=localtime';
3060     }
3061
3062     my $cpu = $nokvm ? "qemu64" : "kvm64";
3063     if (my $cputype = $conf->{cpu}) {
3064         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3065             or die "Cannot parse cpu description: $cputype\n";
3066         $cpu = $cpuconf->{cputype};
3067         $kvm_off = 1 if $cpuconf->{hidden};
3068     }
3069
3070     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3071
3072     push @$cpuFlags , '-x2apic'
3073         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3074
3075     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3076
3077     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3078
3079     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3080
3081         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3082         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3083     }
3084
3085     add_hyperv_enlighments($cpuFlags, $winversion, $machine_type, $kvmver, $nokvm, $conf->{bios}, $gpu_passthrough);
3086
3087     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3088
3089     push @$cpuFlags, 'kvm=off' if $kvm_off;
3090
3091     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3092         die "internal error"; # should not happen
3093
3094     push @$cpuFlags, "vendor=${cpu_vendor}"
3095         if $cpu_vendor ne 'default';
3096
3097     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3098
3099     push @$cmd, '-cpu', $cpu;
3100
3101     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3102
3103     push @$cmd, '-S' if $conf->{freeze};
3104
3105     # set keyboard layout
3106     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3107     push @$cmd, '-k', $kb if $kb;
3108
3109     # enable sound
3110     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3111     #push @$cmd, '-soundhw', 'es1370';
3112     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3113
3114     if($conf->{agent}) {
3115         my $qgasocket = qmp_socket($vmid, 1);
3116         my $pciaddr = print_pci_addr("qga0", $bridges);
3117         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3118         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3119         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3120     }
3121
3122     my $spice_port;
3123
3124     if ($qxlnum) {
3125         if ($qxlnum > 1) {
3126             if ($conf->{ostype} && $conf->{ostype} =~ m/^w/){
3127                 for(my $i = 1; $i < $qxlnum; $i++){
3128                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3129                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3130                 }
3131             } else {
3132                 # assume other OS works like Linux
3133                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3134                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3135             }
3136         }
3137
3138         my $pciaddr = print_pci_addr("spice", $bridges);
3139
3140         my $nodename = PVE::INotify::nodename();
3141         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3142         $spice_port = PVE::Tools::next_spice_port($pfamily);
3143
3144         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=DES-CBC3-SHA,seamless-migration=on";
3145
3146         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3147         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3148         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3149     }
3150
3151     # enable balloon by default, unless explicitly disabled
3152     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3153         $pciaddr = print_pci_addr("balloon0", $bridges);
3154         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3155     }
3156
3157     if ($conf->{watchdog}) {
3158         my $wdopts = parse_watchdog($conf->{watchdog});
3159         $pciaddr = print_pci_addr("watchdog", $bridges);
3160         my $watchdog = $wdopts->{model} || 'i6300esb';
3161         push @$devices, '-device', "$watchdog$pciaddr";
3162         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3163     }
3164
3165     my $vollist = [];
3166     my $scsicontroller = {};
3167     my $ahcicontroller = {};
3168     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3169
3170     # Add iscsi initiator name if available
3171     if (my $initiator = get_initiator_name()) {
3172         push @$devices, '-iscsi', "initiator-name=$initiator";
3173     }
3174
3175     foreach_drive($conf, sub {
3176         my ($ds, $drive) = @_;
3177
3178         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3179             push @$vollist, $drive->{file};
3180         }
3181
3182         $use_virtio = 1 if $ds =~ m/^virtio/;
3183
3184         if (drive_is_cdrom ($drive)) {
3185             if ($bootindex_hash->{d}) {
3186                 $drive->{bootindex} = $bootindex_hash->{d};
3187                 $bootindex_hash->{d} += 1;
3188             }
3189         } else {
3190             if ($bootindex_hash->{c}) {
3191                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3192                 $bootindex_hash->{c} += 1;
3193             }
3194         }
3195
3196         if($drive->{interface} eq 'virtio'){
3197            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3198         }
3199
3200         if ($drive->{interface} eq 'scsi') {
3201
3202             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3203
3204             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3205             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3206
3207             my $iothread = '';
3208             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3209                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3210                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3211             } elsif ($drive->{iothread}) {
3212                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3213             }
3214
3215             my $queues = '';
3216             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3217                 $queues = ",num_queues=$drive->{queues}";
3218             }
3219
3220             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3221             $scsicontroller->{$controller}=1;
3222         }
3223
3224         if ($drive->{interface} eq 'sata') {
3225            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3226            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3227            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3228            $ahcicontroller->{$controller}=1;
3229         }
3230
3231         if ($drive->{interface} eq 'efidisk') {
3232             # this will be added somewhere else
3233             return;
3234         }
3235
3236         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3237         push @$devices, '-drive',$drive_cmd;
3238         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3239     });
3240
3241     for (my $i = 0; $i < $MAX_NETS; $i++) {
3242          next if !$conf->{"net$i"};
3243          my $d = parse_net($conf->{"net$i"});
3244          next if !$d;
3245
3246          $use_virtio = 1 if $d->{model} eq 'virtio';
3247
3248          if ($bootindex_hash->{n}) {
3249             $d->{bootindex} = $bootindex_hash->{n};
3250             $bootindex_hash->{n} += 1;
3251          }
3252
3253          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3254          push @$devices, '-netdev', $netdevfull;
3255
3256          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3257          push @$devices, '-device', $netdevicefull;
3258     }
3259
3260     if (!$q35) {
3261         # add pci bridges
3262         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3263            $bridges->{1} = 1;
3264            $bridges->{2} = 1;
3265         }
3266
3267         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3268
3269         while (my ($k, $v) = each %$bridges) {
3270             $pciaddr = print_pci_addr("pci.$k");
3271             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3272         }
3273     }
3274
3275     # add custom args
3276     if ($conf->{args}) {
3277         my $aa = PVE::Tools::split_args($conf->{args});
3278         push @$cmd, @$aa;
3279     }
3280
3281     push @$cmd, @$devices;
3282     push @$cmd, '-rtc', join(',', @$rtcFlags)
3283         if scalar(@$rtcFlags);
3284     push @$cmd, '-machine', join(',', @$machineFlags)
3285         if scalar(@$machineFlags);
3286     push @$cmd, '-global', join(',', @$globalFlags)
3287         if scalar(@$globalFlags);
3288
3289     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3290 }
3291
3292 sub vnc_socket {
3293     my ($vmid) = @_;
3294     return "${var_run_tmpdir}/$vmid.vnc";
3295 }
3296
3297 sub spice_port {
3298     my ($vmid) = @_;
3299
3300     my $res = vm_mon_cmd($vmid, 'query-spice');
3301
3302     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3303 }
3304
3305 sub qmp_socket {
3306     my ($vmid, $qga) = @_;
3307     my $sockettype = $qga ? 'qga' : 'qmp';
3308     return "${var_run_tmpdir}/$vmid.$sockettype";
3309 }
3310
3311 sub pidfile_name {
3312     my ($vmid) = @_;
3313     return "${var_run_tmpdir}/$vmid.pid";
3314 }
3315
3316 sub vm_devices_list {
3317     my ($vmid) = @_;
3318
3319     my $res = vm_mon_cmd($vmid, 'query-pci');
3320     my $devices = {};
3321     foreach my $pcibus (@$res) {
3322         foreach my $device (@{$pcibus->{devices}}) {
3323             next if !$device->{'qdev_id'};
3324             if ($device->{'pci_bridge'}) {
3325                 $devices->{$device->{'qdev_id'}} = 1;
3326                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3327                     next if !$bridge_device->{'qdev_id'};
3328                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3329                     $devices->{$device->{'qdev_id'}}++;
3330                 }
3331             } else {
3332                 $devices->{$device->{'qdev_id'}} = 1;
3333             }
3334         }
3335     }
3336
3337     my $resblock = vm_mon_cmd($vmid, 'query-block');
3338     foreach my $block (@$resblock) {
3339         if($block->{device} =~ m/^drive-(\S+)/){
3340                 $devices->{$1} = 1;
3341         }
3342     }
3343
3344     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3345     foreach my $mice (@$resmice) {
3346         if ($mice->{name} eq 'QEMU HID Tablet') {
3347             $devices->{tablet} = 1;
3348             last;
3349         }
3350     }
3351
3352     # for usb devices there is no query-usb
3353     # but we can iterate over the entries in
3354     # qom-list path=/machine/peripheral
3355     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3356     foreach my $per (@$resperipheral) {
3357         if ($per->{name} =~ m/^usb\d+$/) {
3358             $devices->{$per->{name}} = 1;
3359         }
3360     }
3361
3362     return $devices;
3363 }
3364
3365 sub vm_deviceplug {
3366     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3367
3368     my $q35 = machine_type_is_q35($conf);
3369
3370     my $devices_list = vm_devices_list($vmid);
3371     return 1 if defined($devices_list->{$deviceid});
3372
3373     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3374
3375     if ($deviceid eq 'tablet') {
3376
3377         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3378
3379     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3380
3381         die "usb hotplug currently not reliable\n";
3382         # since we can't reliably hot unplug all added usb devices
3383         # and usb passthrough disables live migration
3384         # we disable usb hotplugging for now
3385         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3386
3387     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3388
3389         qemu_iothread_add($vmid, $deviceid, $device);
3390
3391         qemu_driveadd($storecfg, $vmid, $device);
3392         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3393
3394         qemu_deviceadd($vmid, $devicefull);
3395         eval { qemu_deviceaddverify($vmid, $deviceid); };
3396         if (my $err = $@) {
3397             eval { qemu_drivedel($vmid, $deviceid); };
3398             warn $@ if $@;
3399             die $err;
3400         }
3401
3402     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3403
3404
3405         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3406         my $pciaddr = print_pci_addr($deviceid);
3407         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3408
3409         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3410
3411         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3412             qemu_iothread_add($vmid, $deviceid, $device);
3413             $devicefull .= ",iothread=iothread-$deviceid";
3414         }
3415
3416         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3417             $devicefull .= ",num_queues=$device->{queues}";
3418         }
3419
3420         qemu_deviceadd($vmid, $devicefull);
3421         qemu_deviceaddverify($vmid, $deviceid);
3422
3423     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3424
3425         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3426         qemu_driveadd($storecfg, $vmid, $device);
3427
3428         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3429         eval { qemu_deviceadd($vmid, $devicefull); };
3430         if (my $err = $@) {
3431             eval { qemu_drivedel($vmid, $deviceid); };
3432             warn $@ if $@;
3433             die $err;
3434         }
3435
3436     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3437
3438         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3439
3440         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3441         my $use_old_bios_files = undef;
3442         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3443
3444         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3445         qemu_deviceadd($vmid, $netdevicefull);
3446         eval { qemu_deviceaddverify($vmid, $deviceid); };
3447         if (my $err = $@) {
3448             eval { qemu_netdevdel($vmid, $deviceid); };
3449             warn $@ if $@;
3450             die $err;
3451         }
3452
3453     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3454
3455         my $bridgeid = $2;
3456         my $pciaddr = print_pci_addr($deviceid);
3457         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3458
3459         qemu_deviceadd($vmid, $devicefull);
3460         qemu_deviceaddverify($vmid, $deviceid);
3461
3462     } else {
3463         die "can't hotplug device '$deviceid'\n";
3464     }
3465
3466     return 1;
3467 }
3468
3469 # fixme: this should raise exceptions on error!
3470 sub vm_deviceunplug {
3471     my ($vmid, $conf, $deviceid) = @_;
3472
3473     my $devices_list = vm_devices_list($vmid);
3474     return 1 if !defined($devices_list->{$deviceid});
3475
3476     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3477
3478     if ($deviceid eq 'tablet') {
3479
3480         qemu_devicedel($vmid, $deviceid);
3481
3482     } elsif ($deviceid =~ m/^usb\d+$/) {
3483
3484         die "usb hotplug currently not reliable\n";
3485         # when unplugging usb devices this way,
3486         # there may be remaining usb controllers/hubs
3487         # so we disable it for now
3488         qemu_devicedel($vmid, $deviceid);
3489         qemu_devicedelverify($vmid, $deviceid);
3490
3491     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3492
3493         qemu_devicedel($vmid, $deviceid);
3494         qemu_devicedelverify($vmid, $deviceid);
3495         qemu_drivedel($vmid, $deviceid);
3496         qemu_iothread_del($conf, $vmid, $deviceid);
3497
3498     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3499
3500         qemu_devicedel($vmid, $deviceid);
3501         qemu_devicedelverify($vmid, $deviceid);
3502         qemu_iothread_del($conf, $vmid, $deviceid);
3503
3504     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3505
3506         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3507         my $device = parse_drive($deviceid, $conf->{$deviceid});
3508         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3509
3510         qemu_devicedel($vmid, $deviceid);
3511         qemu_drivedel($vmid, $deviceid);
3512         qemu_deletescsihw($conf, $vmid, $deviceid);
3513
3514     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3515
3516         qemu_devicedel($vmid, $deviceid);
3517         qemu_devicedelverify($vmid, $deviceid);
3518         qemu_netdevdel($vmid, $deviceid);
3519
3520     } else {
3521         die "can't unplug device '$deviceid'\n";
3522     }
3523
3524     return 1;
3525 }
3526
3527 sub qemu_deviceadd {
3528     my ($vmid, $devicefull) = @_;
3529
3530     $devicefull = "driver=".$devicefull;
3531     my %options =  split(/[=,]/, $devicefull);
3532
3533     vm_mon_cmd($vmid, "device_add" , %options);
3534 }
3535
3536 sub qemu_devicedel {
3537     my ($vmid, $deviceid) = @_;
3538
3539     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3540 }
3541
3542 sub qemu_iothread_add {
3543     my($vmid, $deviceid, $device) = @_;
3544
3545     if ($device->{iothread}) {
3546         my $iothreads = vm_iothreads_list($vmid);
3547         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3548     }
3549 }
3550
3551 sub qemu_iothread_del {
3552     my($conf, $vmid, $deviceid) = @_;
3553
3554     my $device = parse_drive($deviceid, $conf->{$deviceid});
3555     if ($device->{iothread}) {
3556         my $iothreads = vm_iothreads_list($vmid);
3557         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3558     }
3559 }
3560
3561 sub qemu_objectadd {
3562     my($vmid, $objectid, $qomtype) = @_;
3563
3564     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3565
3566     return 1;
3567 }
3568
3569 sub qemu_objectdel {
3570     my($vmid, $objectid) = @_;
3571
3572     vm_mon_cmd($vmid, "object-del", id => $objectid);
3573
3574     return 1;
3575 }
3576
3577 sub qemu_driveadd {
3578     my ($storecfg, $vmid, $device) = @_;
3579
3580     my $drive = print_drive_full($storecfg, $vmid, $device);
3581     $drive =~ s/\\/\\\\/g;
3582     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3583
3584     # If the command succeeds qemu prints: "OK"
3585     return 1 if $ret =~ m/OK/s;
3586
3587     die "adding drive failed: $ret\n";
3588 }
3589
3590 sub qemu_drivedel {
3591     my($vmid, $deviceid) = @_;
3592
3593     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3594     $ret =~ s/^\s+//;
3595
3596     return 1 if $ret eq "";
3597
3598     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3599     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3600
3601     die "deleting drive $deviceid failed : $ret\n";
3602 }
3603
3604 sub qemu_deviceaddverify {
3605     my ($vmid, $deviceid) = @_;
3606
3607     for (my $i = 0; $i <= 5; $i++) {
3608          my $devices_list = vm_devices_list($vmid);
3609          return 1 if defined($devices_list->{$deviceid});
3610          sleep 1;
3611     }
3612
3613     die "error on hotplug device '$deviceid'\n";
3614 }
3615
3616
3617 sub qemu_devicedelverify {
3618     my ($vmid, $deviceid) = @_;
3619
3620     # need to verify that the device is correctly removed as device_del
3621     # is async and empty return is not reliable
3622
3623     for (my $i = 0; $i <= 5; $i++) {
3624          my $devices_list = vm_devices_list($vmid);
3625          return 1 if !defined($devices_list->{$deviceid});
3626          sleep 1;
3627     }
3628
3629     die "error on hot-unplugging device '$deviceid'\n";
3630 }
3631
3632 sub qemu_findorcreatescsihw {
3633     my ($storecfg, $conf, $vmid, $device) = @_;
3634
3635     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3636
3637     my $scsihwid="$controller_prefix$controller";
3638     my $devices_list = vm_devices_list($vmid);
3639
3640     if(!defined($devices_list->{$scsihwid})) {
3641         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3642     }
3643
3644     return 1;
3645 }
3646
3647 sub qemu_deletescsihw {
3648     my ($conf, $vmid, $opt) = @_;
3649
3650     my $device = parse_drive($opt, $conf->{$opt});
3651
3652     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3653         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3654         return 1;
3655     }
3656
3657     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3658
3659     my $devices_list = vm_devices_list($vmid);
3660     foreach my $opt (keys %{$devices_list}) {
3661         if (PVE::QemuServer::is_valid_drivename($opt)) {
3662             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3663             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3664                 return 1;
3665             }
3666         }
3667     }
3668
3669     my $scsihwid="scsihw$controller";
3670
3671     vm_deviceunplug($vmid, $conf, $scsihwid);
3672
3673     return 1;
3674 }
3675
3676 sub qemu_add_pci_bridge {
3677     my ($storecfg, $conf, $vmid, $device) = @_;
3678
3679     my $bridges = {};
3680
3681     my $bridgeid;
3682
3683     print_pci_addr($device, $bridges);
3684
3685     while (my ($k, $v) = each %$bridges) {
3686         $bridgeid = $k;
3687     }
3688     return 1 if !defined($bridgeid) || $bridgeid < 1;
3689
3690     my $bridge = "pci.$bridgeid";
3691     my $devices_list = vm_devices_list($vmid);
3692
3693     if (!defined($devices_list->{$bridge})) {
3694         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3695     }
3696
3697     return 1;
3698 }
3699
3700 sub qemu_set_link_status {
3701     my ($vmid, $device, $up) = @_;
3702
3703     vm_mon_cmd($vmid, "set_link", name => $device,
3704                up => $up ? JSON::true : JSON::false);
3705 }
3706
3707 sub qemu_netdevadd {
3708     my ($vmid, $conf, $device, $deviceid) = @_;
3709
3710     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3711     my %options =  split(/[=,]/, $netdev);
3712
3713     vm_mon_cmd($vmid, "netdev_add",  %options);
3714     return 1;
3715 }
3716
3717 sub qemu_netdevdel {
3718     my ($vmid, $deviceid) = @_;
3719
3720     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3721 }
3722
3723 sub qemu_usb_hotplug {
3724     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3725
3726     return if !$device;
3727
3728     # remove the old one first
3729     vm_deviceunplug($vmid, $conf, $deviceid);
3730
3731     # check if xhci controller is necessary and available
3732     if ($device->{usb3}) {
3733
3734         my $devicelist = vm_devices_list($vmid);
3735
3736         if (!$devicelist->{xhci}) {
3737             my $pciaddr = print_pci_addr("xhci");
3738             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3739         }
3740     }
3741     my $d = parse_usb_device($device->{host});
3742     $d->{usb3} = $device->{usb3};
3743
3744     # add the new one
3745     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3746 }
3747
3748 sub qemu_cpu_hotplug {
3749     my ($vmid, $conf, $vcpus) = @_;
3750
3751     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3752
3753     my $sockets = 1;
3754     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3755     $sockets = $conf->{sockets} if  $conf->{sockets};
3756     my $cores = $conf->{cores} || 1;
3757     my $maxcpus = $sockets * $cores;
3758
3759     $vcpus = $maxcpus if !$vcpus;
3760
3761     die "you can't add more vcpus than maxcpus\n"
3762         if $vcpus > $maxcpus;
3763
3764     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3765
3766     if ($vcpus < $currentvcpus) {
3767
3768         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3769
3770             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3771                 qemu_devicedel($vmid, "cpu$i");
3772                 my $retry = 0;
3773                 my $currentrunningvcpus = undef;
3774                 while (1) {
3775                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3776                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3777                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3778                     $retry++;
3779                     sleep 1;
3780                 }
3781                 #update conf after each succesfull cpu unplug
3782                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3783                 PVE::QemuConfig->write_config($vmid, $conf);
3784             }
3785         } else {
3786             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3787         }
3788
3789         return;
3790     }
3791
3792     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3793     die "vcpus in running vm does not match its configuration\n"
3794         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3795
3796     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3797
3798         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3799             my $cpustr = print_cpu_device($conf, $i);
3800             qemu_deviceadd($vmid, $cpustr);
3801
3802             my $retry = 0;
3803             my $currentrunningvcpus = undef;
3804             while (1) {
3805                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3806                 last if scalar(@{$currentrunningvcpus}) == $i;
3807                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3808                 sleep 1;
3809                 $retry++;
3810             }
3811             #update conf after each succesfull cpu hotplug
3812             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3813             PVE::QemuConfig->write_config($vmid, $conf);
3814         }
3815     } else {
3816
3817         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3818             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3819         }
3820     }
3821 }
3822
3823 sub qemu_block_set_io_throttle {
3824     my ($vmid, $deviceid,
3825         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3826         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3827         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3828         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3829
3830     return if !check_running($vmid) ;
3831
3832     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3833         bps => int($bps),
3834         bps_rd => int($bps_rd),
3835         bps_wr => int($bps_wr),
3836         iops => int($iops),
3837         iops_rd => int($iops_rd),
3838         iops_wr => int($iops_wr),
3839         bps_max => int($bps_max),
3840         bps_rd_max => int($bps_rd_max),
3841         bps_wr_max => int($bps_wr_max),
3842         iops_max => int($iops_max),
3843         iops_rd_max => int($iops_rd_max),
3844         iops_wr_max => int($iops_wr_max),
3845         bps_max_length => int($bps_max_length),
3846         bps_rd_max_length => int($bps_rd_max_length),
3847         bps_wr_max_length => int($bps_wr_max_length),
3848         iops_max_length => int($iops_max_length),
3849         iops_rd_max_length => int($iops_rd_max_length),
3850         iops_wr_max_length => int($iops_wr_max_length),
3851     );
3852
3853 }
3854
3855 # old code, only used to shutdown old VM after update
3856 sub __read_avail {
3857     my ($fh, $timeout) = @_;
3858
3859     my $sel = new IO::Select;
3860     $sel->add($fh);
3861
3862     my $res = '';
3863     my $buf;
3864
3865     my @ready;
3866     while (scalar (@ready = $sel->can_read($timeout))) {
3867         my $count;
3868         if ($count = $fh->sysread($buf, 8192)) {
3869             if ($buf =~ /^(.*)\(qemu\) $/s) {
3870                 $res .= $1;
3871                 last;
3872             } else {
3873                 $res .= $buf;
3874             }
3875         } else {
3876             if (!defined($count)) {
3877                 die "$!\n";
3878             }
3879             last;
3880         }
3881     }
3882
3883     die "monitor read timeout\n" if !scalar(@ready);
3884
3885     return $res;
3886 }
3887
3888 # old code, only used to shutdown old VM after update
3889 sub vm_monitor_command {
3890     my ($vmid, $cmdstr, $nocheck) = @_;
3891
3892     my $res;
3893
3894     eval {
3895         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3896
3897         my $sname = "${var_run_tmpdir}/$vmid.mon";
3898
3899         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3900             die "unable to connect to VM $vmid socket - $!\n";
3901
3902         my $timeout = 3;
3903
3904         # hack: migrate sometime blocks the monitor (when migrate_downtime
3905         # is set)
3906         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3907             $timeout = 60*60; # 1 hour
3908         }
3909
3910         # read banner;
3911         my $data = __read_avail($sock, $timeout);
3912
3913         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3914             die "got unexpected qemu monitor banner\n";
3915         }
3916
3917         my $sel = new IO::Select;
3918         $sel->add($sock);
3919
3920         if (!scalar(my @ready = $sel->can_write($timeout))) {
3921             die "monitor write error - timeout";
3922         }
3923
3924         my $fullcmd = "$cmdstr\r";
3925
3926         # syslog('info', "VM $vmid monitor command: $cmdstr");
3927
3928         my $b;
3929         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3930             die "monitor write error - $!";
3931         }
3932
3933         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3934
3935         $timeout = 20;
3936
3937         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3938             $timeout = 60*60; # 1 hour
3939         } elsif ($cmdstr =~ m/^(eject|change)/) {
3940             $timeout = 60; # note: cdrom mount command is slow
3941         }
3942         if ($res = __read_avail($sock, $timeout)) {
3943
3944             my @lines = split("\r?\n", $res);
3945
3946             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3947
3948             $res = join("\n", @lines);
3949             $res .= "\n";
3950         }
3951     };
3952
3953     my $err = $@;
3954
3955     if ($err) {
3956         syslog("err", "VM $vmid monitor command failed - $err");
3957         die $err;
3958     }
3959
3960     return $res;
3961 }
3962
3963 sub qemu_block_resize {
3964     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3965
3966     my $running = check_running($vmid);
3967
3968     return if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3969
3970     return if !$running;
3971
3972     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3973
3974 }
3975
3976 sub qemu_volume_snapshot {
3977     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3978
3979     my $running = check_running($vmid);
3980
3981     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3982         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3983     } else {
3984         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
3985     }
3986 }
3987
3988 sub qemu_volume_snapshot_delete {
3989     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3990
3991     my $running = check_running($vmid);
3992
3993     return if !PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
3994
3995     return if !$running;
3996
3997     vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
3998 }
3999
4000 sub set_migration_caps {
4001     my ($vmid) = @_;
4002
4003     my $cap_ref = [];
4004
4005     my $enabled_cap = {
4006         "auto-converge" => 1,
4007         "xbzrle" => 1,
4008         "x-rdma-pin-all" => 0,
4009         "zero-blocks" => 0,
4010         "compress" => 0
4011     };
4012
4013     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4014
4015     for my $supported_capability (@$supported_capabilities) {
4016         push @$cap_ref, {
4017             capability => $supported_capability->{capability},
4018             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4019         };
4020     }
4021
4022     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4023 }
4024
4025 my $fast_plug_option = {
4026     'lock' => 1,
4027     'name' => 1,
4028     'onboot' => 1,
4029     'shares' => 1,
4030     'startup' => 1,
4031     'description' => 1,
4032     'protection' => 1,
4033 };
4034
4035 # hotplug changes in [PENDING]
4036 # $selection hash can be used to only apply specified options, for
4037 # example: { cores => 1 } (only apply changed 'cores')
4038 # $errors ref is used to return error messages
4039 sub vmconfig_hotplug_pending {
4040     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4041
4042     my $defaults = load_defaults();
4043
4044     # commit values which do not have any impact on running VM first
4045     # Note: those option cannot raise errors, we we do not care about
4046     # $selection and always apply them.
4047
4048     my $add_error = sub {
4049         my ($opt, $msg) = @_;
4050         $errors->{$opt} = "hotplug problem - $msg";
4051     };
4052
4053     my $changes = 0;
4054     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4055         if ($fast_plug_option->{$opt}) {
4056             $conf->{$opt} = $conf->{pending}->{$opt};
4057             delete $conf->{pending}->{$opt};
4058             $changes = 1;
4059         }
4060     }
4061
4062     if ($changes) {
4063         PVE::QemuConfig->write_config($vmid, $conf);
4064         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4065     }
4066
4067     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4068
4069     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4070     while (my ($opt, $force) = each %$pending_delete_hash) {
4071         next if $selection && !$selection->{$opt};
4072         eval {
4073             if ($opt eq 'hotplug') {
4074                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4075             } elsif ($opt eq 'tablet') {
4076                 die "skip\n" if !$hotplug_features->{usb};
4077                 if ($defaults->{tablet}) {
4078                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4079                 } else {
4080                     vm_deviceunplug($vmid, $conf, $opt);
4081                 }
4082             } elsif ($opt =~ m/^usb\d+/) {
4083                 die "skip\n";
4084                 # since we cannot reliably hot unplug usb devices
4085                 # we are disabling it
4086                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4087                 vm_deviceunplug($vmid, $conf, $opt);
4088             } elsif ($opt eq 'vcpus') {
4089                 die "skip\n" if !$hotplug_features->{cpu};
4090                 qemu_cpu_hotplug($vmid, $conf, undef);
4091             } elsif ($opt eq 'balloon') {
4092                 # enable balloon device is not hotpluggable
4093                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4094             } elsif ($fast_plug_option->{$opt}) {
4095                 # do nothing
4096             } elsif ($opt =~ m/^net(\d+)$/) {
4097                 die "skip\n" if !$hotplug_features->{network};
4098                 vm_deviceunplug($vmid, $conf, $opt);
4099             } elsif (is_valid_drivename($opt)) {
4100                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4101                 vm_deviceunplug($vmid, $conf, $opt);
4102                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4103             } elsif ($opt =~ m/^memory$/) {
4104                 die "skip\n" if !$hotplug_features->{memory};
4105                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4106             } elsif ($opt eq 'cpuunits') {
4107                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4108             } elsif ($opt eq 'cpulimit') {
4109                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4110             } else {
4111                 die "skip\n";
4112             }
4113         };
4114         if (my $err = $@) {
4115             &$add_error($opt, $err) if $err ne "skip\n";
4116         } else {
4117             # save new config if hotplug was successful
4118             delete $conf->{$opt};
4119             vmconfig_undelete_pending_option($conf, $opt);
4120             PVE::QemuConfig->write_config($vmid, $conf);
4121             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4122         }
4123     }
4124
4125     foreach my $opt (keys %{$conf->{pending}}) {
4126         next if $selection && !$selection->{$opt};
4127         my $value = $conf->{pending}->{$opt};
4128         eval {
4129             if ($opt eq 'hotplug') {
4130                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4131             } elsif ($opt eq 'tablet') {
4132                 die "skip\n" if !$hotplug_features->{usb};
4133                 if ($value == 1) {
4134                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4135                 } elsif ($value == 0) {
4136                     vm_deviceunplug($vmid, $conf, $opt);
4137                 }
4138             } elsif ($opt =~ m/^usb\d+$/) {
4139                 die "skip\n";
4140                 # since we cannot reliably hot unplug usb devices
4141                 # we are disabling it
4142                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4143                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4144                 die "skip\n" if !$d;
4145                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4146             } elsif ($opt eq 'vcpus') {
4147                 die "skip\n" if !$hotplug_features->{cpu};
4148                 qemu_cpu_hotplug($vmid, $conf, $value);
4149             } elsif ($opt eq 'balloon') {
4150                 # enable/disable balloning device is not hotpluggable
4151                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4152                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4153                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4154
4155                 # allow manual ballooning if shares is set to zero
4156                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4157                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4158                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4159                 }
4160             } elsif ($opt =~ m/^net(\d+)$/) {
4161                 # some changes can be done without hotplug
4162                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4163                                     $vmid, $opt, $value);
4164             } elsif (is_valid_drivename($opt)) {
4165                 # some changes can be done without hotplug
4166                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4167                                      $vmid, $opt, $value, 1);
4168             } elsif ($opt =~ m/^memory$/) { #dimms
4169                 die "skip\n" if !$hotplug_features->{memory};
4170                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4171             } elsif ($opt eq 'cpuunits') {
4172                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4173             } elsif ($opt eq 'cpulimit') {
4174                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4175                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4176             } else {
4177                 die "skip\n";  # skip non-hot-pluggable options
4178             }
4179         };
4180         if (my $err = $@) {
4181             &$add_error($opt, $err) if $err ne "skip\n";
4182         } else {
4183             # save new config if hotplug was successful
4184             $conf->{$opt} = $value;
4185             delete $conf->{pending}->{$opt};
4186             PVE::QemuConfig->write_config($vmid, $conf);
4187             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4188         }
4189     }
4190 }
4191
4192 sub try_deallocate_drive {
4193     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4194
4195     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4196         my $volid = $drive->{file};
4197         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4198             my $sid = PVE::Storage::parse_volume_id($volid);
4199             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4200
4201             # check if the disk is really unused
4202             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4203                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4204             PVE::Storage::vdisk_free($storecfg, $volid);
4205             return 1;
4206         } else {
4207             # If vm is not owner of this disk remove from config
4208             return 1;
4209         }
4210     }
4211
4212     return undef;
4213 }
4214
4215 sub vmconfig_delete_or_detach_drive {
4216     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4217
4218     my $drive = parse_drive($opt, $conf->{$opt});
4219
4220     my $rpcenv = PVE::RPCEnvironment::get();
4221     my $authuser = $rpcenv->get_user();
4222
4223     if ($force) {
4224         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4225         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4226     } else {
4227         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4228     }
4229 }
4230
4231 sub vmconfig_apply_pending {
4232     my ($vmid, $conf, $storecfg) = @_;
4233
4234     # cold plug
4235
4236     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4237     while (my ($opt, $force) = each %$pending_delete_hash) {
4238         die "internal error" if $opt =~ m/^unused/;
4239         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4240         if (!defined($conf->{$opt})) {
4241             vmconfig_undelete_pending_option($conf, $opt);
4242             PVE::QemuConfig->write_config($vmid, $conf);
4243         } elsif (is_valid_drivename($opt)) {
4244             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4245             vmconfig_undelete_pending_option($conf, $opt);
4246             delete $conf->{$opt};
4247             PVE::QemuConfig->write_config($vmid, $conf);
4248         } else {
4249             vmconfig_undelete_pending_option($conf, $opt);
4250             delete $conf->{$opt};
4251             PVE::QemuConfig->write_config($vmid, $conf);
4252         }
4253     }
4254
4255     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4256
4257     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4258         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4259
4260         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4261             # skip if nothing changed
4262         } elsif (is_valid_drivename($opt)) {
4263             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4264                 if defined($conf->{$opt});
4265             $conf->{$opt} = $conf->{pending}->{$opt};
4266         } else {
4267             $conf->{$opt} = $conf->{pending}->{$opt};
4268         }
4269
4270         delete $conf->{pending}->{$opt};
4271         PVE::QemuConfig->write_config($vmid, $conf);
4272     }
4273 }
4274
4275 my $safe_num_ne = sub {
4276     my ($a, $b) = @_;
4277
4278     return 0 if !defined($a) && !defined($b);
4279     return 1 if !defined($a);
4280     return 1 if !defined($b);
4281
4282     return $a != $b;
4283 };
4284
4285 my $safe_string_ne = sub {
4286     my ($a, $b) = @_;
4287
4288     return 0 if !defined($a) && !defined($b);
4289     return 1 if !defined($a);
4290     return 1 if !defined($b);
4291
4292     return $a ne $b;
4293 };
4294
4295 sub vmconfig_update_net {
4296     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4297
4298     my $newnet = parse_net($value);
4299
4300     if ($conf->{$opt}) {
4301         my $oldnet = parse_net($conf->{$opt});
4302
4303         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4304             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4305             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4306             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4307
4308             # for non online change, we try to hot-unplug
4309             die "skip\n" if !$hotplug;
4310             vm_deviceunplug($vmid, $conf, $opt);
4311         } else {
4312
4313             die "internal error" if $opt !~ m/net(\d+)/;
4314             my $iface = "tap${vmid}i$1";
4315
4316             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4317                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4318                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4319                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4320                 PVE::Network::tap_unplug($iface);
4321                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4322             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4323                 # Rate can be applied on its own but any change above needs to
4324                 # include the rate in tap_plug since OVS resets everything.
4325                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4326             }
4327
4328             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4329                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4330             }
4331
4332             return 1;
4333         }
4334     }
4335
4336     if ($hotplug) {
4337         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4338     } else {
4339         die "skip\n";
4340     }
4341 }
4342
4343 sub vmconfig_update_disk {
4344     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4345
4346     # fixme: do we need force?
4347
4348     my $drive = parse_drive($opt, $value);
4349
4350     if ($conf->{$opt}) {
4351
4352         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4353
4354             my $media = $drive->{media} || 'disk';
4355             my $oldmedia = $old_drive->{media} || 'disk';
4356             die "unable to change media type\n" if $media ne $oldmedia;
4357
4358             if (!drive_is_cdrom($old_drive)) {
4359
4360                 if ($drive->{file} ne $old_drive->{file}) {
4361
4362                     die "skip\n" if !$hotplug;
4363
4364                     # unplug and register as unused
4365                     vm_deviceunplug($vmid, $conf, $opt);
4366                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4367
4368                 } else {
4369                     # update existing disk
4370
4371                     # skip non hotpluggable value
4372                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4373                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4374                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4375                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4376                         die "skip\n";
4377                     }
4378
4379                     # apply throttle
4380                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4381                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4382                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4383                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4384                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4385                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4386                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4387                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4388                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4389                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4390                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4391                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4392                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4393                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4394                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4395                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4396                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4397                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4398
4399                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4400                                                    ($drive->{mbps} || 0)*1024*1024,
4401                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4402                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4403                                                    $drive->{iops} || 0,
4404                                                    $drive->{iops_rd} || 0,
4405                                                    $drive->{iops_wr} || 0,
4406                                                    ($drive->{mbps_max} || 0)*1024*1024,
4407                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4408                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4409                                                    $drive->{iops_max} || 0,
4410                                                    $drive->{iops_rd_max} || 0,
4411                                                    $drive->{iops_wr_max} || 0,
4412                                                    $drive->{bps_max_length} || 1,
4413                                                    $drive->{bps_rd_max_length} || 1,
4414                                                    $drive->{bps_wr_max_length} || 1,
4415                                                    $drive->{iops_max_length} || 1,
4416                                                    $drive->{iops_rd_max_length} || 1,
4417                                                    $drive->{iops_wr_max_length} || 1);
4418
4419                     }
4420
4421                     return 1;
4422                 }
4423
4424             } else { # cdrom
4425
4426                 if ($drive->{file} eq 'none') {
4427                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4428                 } else {
4429                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4430                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4431                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4432                 }
4433
4434                 return 1;
4435             }
4436         }
4437     }
4438
4439     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4440     # hotplug new disks
4441     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4442     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4443 }
4444
4445 sub vm_start {
4446     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4447         $forcemachine, $spice_ticket, $migration_network, $migration_type) = @_;
4448
4449     PVE::QemuConfig->lock_config($vmid, sub {
4450         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4451
4452         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4453
4454         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4455
4456         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4457
4458         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4459             vmconfig_apply_pending($vmid, $conf, $storecfg);
4460             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4461         }
4462
4463         my $defaults = load_defaults();
4464
4465         # set environment variable useful inside network script
4466         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4467
4468         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4469
4470         my $migrate_port = 0;
4471         my $migrate_uri;
4472         if ($statefile) {
4473             if ($statefile eq 'tcp') {
4474                 my $localip = "localhost";
4475                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4476                 my $nodename = PVE::INotify::nodename();
4477
4478                 if ($migration_type eq 'insecure') {
4479                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4480                     if ($migrate_network_addr) {
4481                         $localip = $migrate_network_addr;
4482                     } else {
4483                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4484                     }
4485
4486                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4487                 }
4488
4489                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4490                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4491                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4492                 push @$cmd, '-incoming', $migrate_uri;
4493                 push @$cmd, '-S';
4494
4495             } elsif ($statefile eq 'unix') {
4496                 # should be default for secure migrations as a ssh TCP forward
4497                 # tunnel is not deterministic reliable ready and fails regurarly
4498                 # to set up in time, so use UNIX socket forwards
4499                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4500                 unlink $socket_addr;
4501
4502                 $migrate_uri = "unix:$socket_addr";
4503
4504                 push @$cmd, '-incoming', $migrate_uri;
4505                 push @$cmd, '-S';
4506
4507             } else {
4508                 push @$cmd, '-loadstate', $statefile;
4509             }
4510         } elsif ($paused) {
4511             push @$cmd, '-S';
4512         }
4513
4514         # host pci devices
4515         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4516           my $d = parse_hostpci($conf->{"hostpci$i"});
4517           next if !$d;
4518           my $pcidevices = $d->{pciid};
4519           foreach my $pcidevice (@$pcidevices) {
4520                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4521
4522                 my $info = pci_device_info("0000:$pciid");
4523                 die "IOMMU not present\n" if !check_iommu_support();
4524                 die "no pci device info for device '$pciid'\n" if !$info;
4525                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4526                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4527           }
4528         }
4529
4530         PVE::Storage::activate_volumes($storecfg, $vollist);
4531
4532         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4533             my $cmd = [];
4534             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4535             eval  { run_command($cmd); };
4536         }
4537
4538         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4539                                                   : $defaults->{cpuunits};
4540
4541         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4542
4543         my %properties = (
4544             Slice => 'qemu.slice',
4545             KillMode => 'none',
4546             CPUShares => $cpuunits
4547         );
4548
4549         if (my $cpulimit = $conf->{cpulimit}) {
4550             $properties{CPUQuota} = int($cpulimit * 100);
4551         }
4552         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4553
4554         if ($conf->{hugepages}) {
4555
4556             my $code = sub {
4557                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4558                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4559
4560                 PVE::QemuServer::Memory::hugepages_mount();
4561                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4562
4563                 eval  {
4564                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4565                     run_command($cmd, %run_params);
4566                 };
4567
4568                 if (my $err = $@) {
4569                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4570                     die $err;
4571                 }
4572
4573                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4574             };
4575             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4576
4577         } else {
4578             eval  {
4579                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4580                 run_command($cmd, %run_params);
4581             };
4582         }
4583
4584         if (my $err = $@) {
4585             # deactivate volumes if start fails
4586             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4587             die "start failed: $err";
4588         }
4589
4590         print "migration listens on $migrate_uri\n" if $migrate_uri;
4591
4592         if ($statefile && $statefile ne 'tcp')  {
4593             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4594             warn $@ if $@;
4595         }
4596
4597         if ($migratedfrom) {
4598
4599             eval {
4600                 set_migration_caps($vmid);
4601             };
4602             warn $@ if $@;
4603
4604             if ($spice_port) {
4605                 print "spice listens on port $spice_port\n";
4606                 if ($spice_ticket) {
4607                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4608                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4609                 }
4610             }
4611
4612         } else {
4613
4614             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4615                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4616                     if $conf->{balloon};
4617             }
4618
4619             foreach my $opt (keys %$conf) {
4620                 next if $opt !~  m/^net\d+$/;
4621                 my $nicconf = parse_net($conf->{$opt});
4622                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4623             }
4624         }
4625
4626         vm_mon_cmd_nocheck($vmid, 'qom-set',
4627                     path => "machine/peripheral/balloon0",
4628                     property => "guest-stats-polling-interval",
4629                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4630
4631     });
4632 }
4633
4634 sub vm_mon_cmd {
4635     my ($vmid, $execute, %params) = @_;
4636
4637     my $cmd = { execute => $execute, arguments => \%params };
4638     vm_qmp_command($vmid, $cmd);
4639 }
4640
4641 sub vm_mon_cmd_nocheck {
4642     my ($vmid, $execute, %params) = @_;
4643
4644     my $cmd = { execute => $execute, arguments => \%params };
4645     vm_qmp_command($vmid, $cmd, 1);
4646 }
4647
4648 sub vm_qmp_command {
4649     my ($vmid, $cmd, $nocheck) = @_;
4650
4651     my $res;
4652
4653     my $timeout;
4654     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4655         $timeout = $cmd->{arguments}->{timeout};
4656         delete $cmd->{arguments}->{timeout};
4657     }
4658
4659     eval {
4660         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4661         my $sname = qmp_socket($vmid);
4662         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4663             my $qmpclient = PVE::QMPClient->new();
4664
4665             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4666         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4667             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4668                 if scalar(%{$cmd->{arguments}});
4669             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4670         } else {
4671             die "unable to open monitor socket\n";
4672         }
4673     };
4674     if (my $err = $@) {
4675         syslog("err", "VM $vmid qmp command failed - $err");
4676         die $err;
4677     }
4678
4679     return $res;
4680 }
4681
4682 sub vm_human_monitor_command {
4683     my ($vmid, $cmdline) = @_;
4684
4685     my $res;
4686
4687     my $cmd = {
4688         execute => 'human-monitor-command',
4689         arguments => { 'command-line' => $cmdline},
4690     };
4691
4692     return vm_qmp_command($vmid, $cmd);
4693 }
4694
4695 sub vm_commandline {
4696     my ($storecfg, $vmid) = @_;
4697
4698     my $conf = PVE::QemuConfig->load_config($vmid);
4699
4700     my $defaults = load_defaults();
4701
4702     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4703
4704     return PVE::Tools::cmd2string($cmd);
4705 }
4706
4707 sub vm_reset {
4708     my ($vmid, $skiplock) = @_;
4709
4710     PVE::QemuConfig->lock_config($vmid, sub {
4711
4712         my $conf = PVE::QemuConfig->load_config($vmid);
4713
4714         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4715
4716         vm_mon_cmd($vmid, "system_reset");
4717     });
4718 }
4719
4720 sub get_vm_volumes {
4721     my ($conf) = @_;
4722
4723     my $vollist = [];
4724     foreach_volid($conf, sub {
4725         my ($volid, $is_cdrom) = @_;
4726
4727         return if $volid =~ m|^/|;
4728
4729         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4730         return if !$sid;
4731
4732         push @$vollist, $volid;
4733     });
4734
4735     return $vollist;
4736 }
4737
4738 sub vm_stop_cleanup {
4739     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4740
4741     eval {
4742
4743         if (!$keepActive) {
4744             my $vollist = get_vm_volumes($conf);
4745             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4746         }
4747
4748         foreach my $ext (qw(mon qmp pid vnc qga)) {
4749             unlink "/var/run/qemu-server/${vmid}.$ext";
4750         }
4751
4752         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4753     };
4754     warn $@ if $@; # avoid errors - just warn
4755 }
4756
4757 # Note: use $nockeck to skip tests if VM configuration file exists.
4758 # We need that when migration VMs to other nodes (files already moved)
4759 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4760 sub vm_stop {
4761     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4762
4763     $force = 1 if !defined($force) && !$shutdown;
4764
4765     if ($migratedfrom){
4766         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4767         kill 15, $pid if $pid;
4768         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4769         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4770         return;
4771     }
4772
4773     PVE::QemuConfig->lock_config($vmid, sub {
4774
4775         my $pid = check_running($vmid, $nocheck);
4776         return if !$pid;
4777
4778         my $conf;
4779         if (!$nocheck) {
4780             $conf = PVE::QemuConfig->load_config($vmid);
4781             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4782             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4783                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4784                 $timeout = $opts->{down} if $opts->{down};
4785             }
4786         }
4787
4788         $timeout = 60 if !defined($timeout);
4789
4790         eval {
4791             if ($shutdown) {
4792                 if (defined($conf) && $conf->{agent}) {
4793                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4794                 } else {
4795                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4796                 }
4797             } else {
4798                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4799             }
4800         };
4801         my $err = $@;
4802
4803         if (!$err) {
4804             my $count = 0;
4805             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4806                 $count++;
4807                 sleep 1;
4808             }
4809
4810             if ($count >= $timeout) {
4811                 if ($force) {
4812                     warn "VM still running - terminating now with SIGTERM\n";
4813                     kill 15, $pid;
4814                 } else {
4815                     die "VM quit/powerdown failed - got timeout\n";
4816                 }
4817             } else {
4818                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4819                 return;
4820             }
4821         } else {
4822             if ($force) {
4823                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4824                 kill 15, $pid;
4825             } else {
4826                 die "VM quit/powerdown failed\n";
4827             }
4828         }
4829
4830         # wait again
4831         $timeout = 10;
4832
4833         my $count = 0;
4834         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4835             $count++;
4836             sleep 1;
4837         }
4838
4839         if ($count >= $timeout) {
4840             warn "VM still running - terminating now with SIGKILL\n";
4841             kill 9, $pid;
4842             sleep 1;
4843         }
4844
4845         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4846    });
4847 }
4848
4849 sub vm_suspend {
4850     my ($vmid, $skiplock) = @_;
4851
4852     PVE::QemuConfig->lock_config($vmid, sub {
4853
4854         my $conf = PVE::QemuConfig->load_config($vmid);
4855
4856         PVE::QemuConfig->check_lock($conf)
4857             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4858
4859         vm_mon_cmd($vmid, "stop");
4860     });
4861 }
4862
4863 sub vm_resume {
4864     my ($vmid, $skiplock, $nocheck) = @_;
4865
4866     PVE::QemuConfig->lock_config($vmid, sub {
4867
4868         if (!$nocheck) {
4869
4870             my $conf = PVE::QemuConfig->load_config($vmid);
4871
4872             PVE::QemuConfig->check_lock($conf)
4873                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4874
4875             vm_mon_cmd($vmid, "cont");
4876
4877         } else {
4878             vm_mon_cmd_nocheck($vmid, "cont");
4879         }
4880     });
4881 }
4882
4883 sub vm_sendkey {
4884     my ($vmid, $skiplock, $key) = @_;
4885
4886     PVE::QemuConfig->lock_config($vmid, sub {
4887
4888         my $conf = PVE::QemuConfig->load_config($vmid);
4889
4890         # there is no qmp command, so we use the human monitor command
4891         vm_human_monitor_command($vmid, "sendkey $key");
4892     });
4893 }
4894
4895 sub vm_destroy {
4896     my ($storecfg, $vmid, $skiplock) = @_;
4897
4898     PVE::QemuConfig->lock_config($vmid, sub {
4899
4900         my $conf = PVE::QemuConfig->load_config($vmid);
4901
4902         if (!check_running($vmid)) {
4903             destroy_vm($storecfg, $vmid, undef, $skiplock);
4904         } else {
4905             die "VM $vmid is running - destroy failed\n";
4906         }
4907     });
4908 }
4909
4910 # pci helpers
4911
4912 sub file_write {
4913     my ($filename, $buf) = @_;
4914
4915     my $fh = IO::File->new($filename, "w");
4916     return undef if !$fh;
4917
4918     my $res = print $fh $buf;
4919
4920     $fh->close();
4921
4922     return $res;
4923 }
4924
4925 sub pci_device_info {
4926     my ($name) = @_;
4927
4928     my $res;
4929
4930     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
4931     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
4932
4933     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
4934     return undef if !defined($irq) || $irq !~ m/^\d+$/;
4935
4936     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
4937     return undef if !defined($vendor) || $vendor !~ s/^0x//;
4938
4939     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
4940     return undef if !defined($product) || $product !~ s/^0x//;
4941
4942     $res = {
4943         name => $name,
4944         vendor => $vendor,
4945         product => $product,
4946         domain => $domain,
4947         bus => $bus,
4948         slot => $slot,
4949         func => $func,
4950         irq => $irq,
4951         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
4952     };
4953
4954     return $res;
4955 }
4956
4957 sub pci_dev_reset {
4958     my ($dev) = @_;
4959
4960     my $name = $dev->{name};
4961
4962     my $fn = "$pcisysfs/devices/$name/reset";
4963
4964     return file_write($fn, "1");
4965 }
4966
4967 sub pci_dev_bind_to_vfio {
4968     my ($dev) = @_;
4969
4970     my $name = $dev->{name};
4971
4972     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
4973
4974     if (!-d $vfio_basedir) {
4975         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
4976     }
4977     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
4978
4979     my $testdir = "$vfio_basedir/$name";
4980     return 1 if -d $testdir;
4981
4982     my $data = "$dev->{vendor} $dev->{product}";
4983     return undef if !file_write("$vfio_basedir/new_id", $data);
4984
4985     my $fn = "$pcisysfs/devices/$name/driver/unbind";
4986     if (!file_write($fn, $name)) {
4987         return undef if -f $fn;
4988     }
4989
4990     $fn = "$vfio_basedir/bind";
4991     if (! -d $testdir) {
4992         return undef if !file_write($fn, $name);
4993     }
4994
4995     return -d $testdir;
4996 }
4997
4998 sub pci_dev_group_bind_to_vfio {
4999     my ($pciid) = @_;
5000
5001     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5002
5003     if (!-d $vfio_basedir) {
5004         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5005     }
5006     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5007
5008     # get IOMMU group devices
5009     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5010       my @devs = grep /^0000:/, readdir($D);
5011     closedir($D);
5012
5013     foreach my $pciid (@devs) {
5014         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5015
5016         # pci bridges, switches or root ports are not supported
5017         # they have a pci_bus subdirectory so skip them
5018         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5019
5020         my $info = pci_device_info($1);
5021         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5022     }
5023
5024     return 1;
5025 }
5026
5027 # vzdump restore implementaion
5028
5029 sub tar_archive_read_firstfile {
5030     my $archive = shift;
5031
5032     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5033
5034     # try to detect archive type first
5035     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5036         die "unable to open file '$archive'\n";
5037     my $firstfile = <$fh>;
5038     kill 15, $pid;
5039     close $fh;
5040
5041     die "ERROR: archive contaions no data\n" if !$firstfile;
5042     chomp $firstfile;
5043
5044     return $firstfile;
5045 }
5046
5047 sub tar_restore_cleanup {
5048     my ($storecfg, $statfile) = @_;
5049
5050     print STDERR "starting cleanup\n";
5051
5052     if (my $fd = IO::File->new($statfile, "r")) {
5053         while (defined(my $line = <$fd>)) {
5054             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5055                 my $volid = $2;
5056                 eval {
5057                     if ($volid =~ m|^/|) {
5058                         unlink $volid || die 'unlink failed\n';
5059                     } else {
5060                         PVE::Storage::vdisk_free($storecfg, $volid);
5061                     }
5062                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5063                 };
5064                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5065             } else {
5066                 print STDERR "unable to parse line in statfile - $line";
5067             }
5068         }
5069         $fd->close();
5070     }
5071 }
5072
5073 sub restore_archive {
5074     my ($archive, $vmid, $user, $opts) = @_;
5075
5076     my $format = $opts->{format};
5077     my $comp;
5078
5079     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5080         $format = 'tar' if !$format;
5081         $comp = 'gzip';
5082     } elsif ($archive =~ m/\.tar$/) {
5083         $format = 'tar' if !$format;
5084     } elsif ($archive =~ m/.tar.lzo$/) {
5085         $format = 'tar' if !$format;
5086         $comp = 'lzop';
5087     } elsif ($archive =~ m/\.vma$/) {
5088         $format = 'vma' if !$format;
5089     } elsif ($archive =~ m/\.vma\.gz$/) {
5090         $format = 'vma' if !$format;
5091         $comp = 'gzip';
5092     } elsif ($archive =~ m/\.vma\.lzo$/) {
5093         $format = 'vma' if !$format;
5094         $comp = 'lzop';
5095     } else {
5096         $format = 'vma' if !$format; # default
5097     }
5098
5099     # try to detect archive format
5100     if ($format eq 'tar') {
5101         return restore_tar_archive($archive, $vmid, $user, $opts);
5102     } else {
5103         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5104     }
5105 }
5106
5107 sub restore_update_config_line {
5108     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5109
5110     return if $line =~ m/^\#qmdump\#/;
5111     return if $line =~ m/^\#vzdump\#/;
5112     return if $line =~ m/^lock:/;
5113     return if $line =~ m/^unused\d+:/;
5114     return if $line =~ m/^parent:/;
5115     return if $line =~ m/^template:/; # restored VM is never a template
5116
5117     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5118     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5119         # try to convert old 1.X settings
5120         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5121         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5122             my ($model, $macaddr) = split(/\=/, $devconfig);
5123             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5124             my $net = {
5125                 model => $model,
5126                 bridge => "vmbr$ind",
5127                 macaddr => $macaddr,
5128             };
5129             my $netstr = print_net($net);
5130
5131             print $outfd "net$cookie->{netcount}: $netstr\n";
5132             $cookie->{netcount}++;
5133         }
5134     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5135         my ($id, $netstr) = ($1, $2);
5136         my $net = parse_net($netstr);
5137         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5138         $netstr = print_net($net);
5139         print $outfd "$id: $netstr\n";
5140     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5141         my $virtdev = $1;
5142         my $value = $3;
5143         my $di = parse_drive($virtdev, $value);
5144         if (defined($di->{backup}) && !$di->{backup}) {
5145             print $outfd "#$line";
5146         } elsif ($map->{$virtdev}) {
5147             delete $di->{format}; # format can change on restore
5148             $di->{file} = $map->{$virtdev};
5149             $value = print_drive($vmid, $di);
5150             print $outfd "$virtdev: $value\n";
5151         } else {
5152             print $outfd $line;
5153         }
5154     } else {
5155         print $outfd $line;
5156     }
5157 }
5158
5159 sub scan_volids {
5160     my ($cfg, $vmid) = @_;
5161
5162     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5163
5164     my $volid_hash = {};
5165     foreach my $storeid (keys %$info) {
5166         foreach my $item (@{$info->{$storeid}}) {
5167             next if !($item->{volid} && $item->{size});
5168             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5169             $volid_hash->{$item->{volid}} = $item;
5170         }
5171     }
5172
5173     return $volid_hash;
5174 }
5175
5176 sub is_volume_in_use {
5177     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5178
5179     my $path = PVE::Storage::path($storecfg, $volid);
5180
5181     my $scan_config = sub {
5182         my ($cref, $snapname) = @_;
5183
5184         foreach my $key (keys %$cref) {
5185             my $value = $cref->{$key};
5186             if (is_valid_drivename($key)) {
5187                 next if $skip_drive && $key eq $skip_drive;
5188                 my $drive = parse_drive($key, $value);
5189                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5190                 return 1 if $volid eq $drive->{file};
5191                 if ($drive->{file} =~ m!^/!) {
5192                     return 1 if $drive->{file} eq $path;
5193                 } else {
5194                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5195                     next if !$storeid;
5196                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5197                     next if !$scfg;
5198                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5199                 }
5200             }
5201         }
5202
5203         return 0;
5204     };
5205
5206     return 1 if &$scan_config($conf);
5207
5208     undef $skip_drive;
5209
5210     foreach my $snapname (keys %{$conf->{snapshots}}) {
5211         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5212     }
5213
5214     return 0;
5215 }
5216
5217 sub update_disksize {
5218     my ($vmid, $conf, $volid_hash) = @_;
5219
5220     my $changes;
5221
5222     my $used = {};
5223
5224     # Note: it is allowed to define multiple storages with same path (alias), so
5225     # we need to check both 'volid' and real 'path' (two different volid can point
5226     # to the same path).
5227
5228     my $usedpath = {};
5229
5230     # update size info
5231     foreach my $opt (keys %$conf) {
5232         if (is_valid_drivename($opt)) {
5233             my $drive = parse_drive($opt, $conf->{$opt});
5234             my $volid = $drive->{file};
5235             next if !$volid;
5236
5237             $used->{$volid} = 1;
5238             if ($volid_hash->{$volid} &&
5239                 (my $path = $volid_hash->{$volid}->{path})) {
5240                 $usedpath->{$path} = 1;
5241             }
5242
5243             next if drive_is_cdrom($drive);
5244             next if !$volid_hash->{$volid};
5245
5246             $drive->{size} = $volid_hash->{$volid}->{size};
5247             my $new = print_drive($vmid, $drive);
5248             if ($new ne $conf->{$opt}) {
5249                 $changes = 1;
5250                 $conf->{$opt} = $new;
5251             }
5252         }
5253     }
5254
5255     # remove 'unusedX' entry if volume is used
5256     foreach my $opt (keys %$conf) {
5257         next if $opt !~ m/^unused\d+$/;
5258         my $volid = $conf->{$opt};
5259         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5260         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5261             $changes = 1;
5262             delete $conf->{$opt};
5263         }
5264     }
5265
5266     foreach my $volid (sort keys %$volid_hash) {
5267         next if $volid =~ m/vm-$vmid-state-/;
5268         next if $used->{$volid};
5269         my $path = $volid_hash->{$volid}->{path};
5270         next if !$path; # just to be sure
5271         next if $usedpath->{$path};
5272         $changes = 1;
5273         PVE::QemuConfig->add_unused_volume($conf, $volid);
5274         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5275     }
5276
5277     return $changes;
5278 }
5279
5280 sub rescan {
5281     my ($vmid, $nolock) = @_;
5282
5283     my $cfg = PVE::Storage::config();
5284
5285     my $volid_hash = scan_volids($cfg, $vmid);
5286
5287     my $updatefn =  sub {
5288         my ($vmid) = @_;
5289
5290         my $conf = PVE::QemuConfig->load_config($vmid);
5291
5292         PVE::QemuConfig->check_lock($conf);
5293
5294         my $vm_volids = {};
5295         foreach my $volid (keys %$volid_hash) {
5296             my $info = $volid_hash->{$volid};
5297             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5298         }
5299
5300         my $changes = update_disksize($vmid, $conf, $vm_volids);
5301
5302         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5303     };
5304
5305     if (defined($vmid)) {
5306         if ($nolock) {
5307             &$updatefn($vmid);
5308         } else {
5309             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5310         }
5311     } else {
5312         my $vmlist = config_list();
5313         foreach my $vmid (keys %$vmlist) {
5314             if ($nolock) {
5315                 &$updatefn($vmid);
5316             } else {
5317                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5318             }
5319         }
5320     }
5321 }
5322
5323 sub restore_vma_archive {
5324     my ($archive, $vmid, $user, $opts, $comp) = @_;
5325
5326     my $input = $archive eq '-' ? "<&STDIN" : undef;
5327     my $readfrom = $archive;
5328
5329     my $uncomp = '';
5330     if ($comp) {
5331         $readfrom = '-';
5332         my $qarchive = PVE::Tools::shellquote($archive);
5333         if ($comp eq 'gzip') {
5334             $uncomp = "zcat $qarchive|";
5335         } elsif ($comp eq 'lzop') {
5336             $uncomp = "lzop -d -c $qarchive|";
5337         } else {
5338             die "unknown compression method '$comp'\n";
5339         }
5340
5341     }
5342
5343     my $tmpdir = "/var/tmp/vzdumptmp$$";
5344     rmtree $tmpdir;
5345
5346     # disable interrupts (always do cleanups)
5347     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5348         warn "got interrupt - ignored\n";
5349     };
5350
5351     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5352     POSIX::mkfifo($mapfifo, 0600);
5353     my $fifofh;
5354
5355     my $openfifo = sub {
5356         open($fifofh, '>', $mapfifo) || die $!;
5357     };
5358
5359     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5360
5361     my $oldtimeout;
5362     my $timeout = 5;
5363
5364     my $devinfo = {};
5365
5366     my $rpcenv = PVE::RPCEnvironment::get();
5367
5368     my $conffile = PVE::QemuConfig->config_file($vmid);
5369     my $tmpfn = "$conffile.$$.tmp";
5370
5371     # Note: $oldconf is undef if VM does not exists
5372     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5373     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5374
5375     my $print_devmap = sub {
5376         my $virtdev_hash = {};
5377
5378         my $cfgfn = "$tmpdir/qemu-server.conf";
5379
5380         # we can read the config - that is already extracted
5381         my $fh = IO::File->new($cfgfn, "r") ||
5382             "unable to read qemu-server.conf - $!\n";
5383
5384         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5385         if (-f $fwcfgfn) {
5386             my $pve_firewall_dir = '/etc/pve/firewall';
5387             mkdir $pve_firewall_dir; # make sure the dir exists
5388             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5389         }
5390
5391         while (defined(my $line = <$fh>)) {
5392             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5393                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5394                 die "archive does not contain data for drive '$virtdev'\n"
5395                     if !$devinfo->{$devname};
5396                 if (defined($opts->{storage})) {
5397                     $storeid = $opts->{storage} || 'local';
5398                 } elsif (!$storeid) {
5399                     $storeid = 'local';
5400                 }
5401                 $format = 'raw' if !$format;
5402                 $devinfo->{$devname}->{devname} = $devname;
5403                 $devinfo->{$devname}->{virtdev} = $virtdev;
5404                 $devinfo->{$devname}->{format} = $format;
5405                 $devinfo->{$devname}->{storeid} = $storeid;
5406
5407                 # check permission on storage
5408                 my $pool = $opts->{pool}; # todo: do we need that?
5409                 if ($user ne 'root@pam') {
5410                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5411                 }
5412
5413                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5414             }
5415         }
5416
5417         foreach my $devname (keys %$devinfo) {
5418             die "found no device mapping information for device '$devname'\n"
5419                 if !$devinfo->{$devname}->{virtdev};
5420         }
5421
5422         my $cfg = PVE::Storage::config();
5423
5424         # create empty/temp config
5425         if ($oldconf) {
5426             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5427             foreach_drive($oldconf, sub {
5428                 my ($ds, $drive) = @_;
5429
5430                 return if drive_is_cdrom($drive);
5431
5432                 my $volid = $drive->{file};
5433
5434                 return if !$volid || $volid =~ m|^/|;
5435
5436                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5437                 return if !$path || !$owner || ($owner != $vmid);
5438
5439                 # Note: only delete disk we want to restore
5440                 # other volumes will become unused
5441                 if ($virtdev_hash->{$ds}) {
5442                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5443                     if (my $err = $@) {
5444                         warn $err;
5445                     }
5446                 }
5447             });
5448
5449             # delete vmstate files
5450             # since after the restore we have no snapshots anymore
5451             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5452                 my $snap = $oldconf->{snapshots}->{$snapname};
5453                 if ($snap->{vmstate}) {
5454                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5455                     if (my $err = $@) {
5456                         warn $err;
5457                     }
5458                 }
5459             }
5460         }
5461
5462         my $map = {};
5463         foreach my $virtdev (sort keys %$virtdev_hash) {
5464             my $d = $virtdev_hash->{$virtdev};
5465             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5466             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5467
5468             # test if requested format is supported
5469             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5470             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5471             $d->{format} = $defFormat if !$supported;
5472
5473             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5474                                                   $d->{format}, undef, $alloc_size);
5475             print STDERR "new volume ID is '$volid'\n";
5476             $d->{volid} = $volid;
5477             my $path = PVE::Storage::path($cfg, $volid);
5478
5479             PVE::Storage::activate_volumes($cfg,[$volid]);
5480
5481             my $write_zeros = 1;
5482             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5483                 $write_zeros = 0;
5484             }
5485
5486             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5487
5488             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5489             $map->{$virtdev} = $volid;
5490         }
5491
5492         $fh->seek(0, 0) || die "seek failed - $!\n";
5493
5494         my $outfd = new IO::File ($tmpfn, "w") ||
5495             die "unable to write config for VM $vmid\n";
5496
5497         my $cookie = { netcount => 0 };
5498         while (defined(my $line = <$fh>)) {
5499             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5500         }
5501
5502         $fh->close();
5503         $outfd->close();
5504     };
5505
5506     eval {
5507         # enable interrupts
5508         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5509             die "interrupted by signal\n";
5510         };
5511         local $SIG{ALRM} = sub { die "got timeout\n"; };
5512
5513         $oldtimeout = alarm($timeout);
5514
5515         my $parser = sub {
5516             my $line = shift;
5517
5518             print "$line\n";
5519
5520             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5521                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5522                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5523             } elsif ($line =~ m/^CTIME: /) {
5524                 # we correctly received the vma config, so we can disable
5525                 # the timeout now for disk allocation (set to 10 minutes, so
5526                 # that we always timeout if something goes wrong)
5527                 alarm(600);
5528                 &$print_devmap();
5529                 print $fifofh "done\n";
5530                 my $tmp = $oldtimeout || 0;
5531                 $oldtimeout = undef;
5532                 alarm($tmp);
5533                 close($fifofh);
5534             }
5535         };
5536
5537         print "restore vma archive: $cmd\n";
5538         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5539     };
5540     my $err = $@;
5541
5542     alarm($oldtimeout) if $oldtimeout;
5543
5544     my $vollist = [];
5545     foreach my $devname (keys %$devinfo) {
5546         my $volid = $devinfo->{$devname}->{volid};
5547         push @$vollist, $volid if $volid;
5548     }
5549
5550     my $cfg = PVE::Storage::config();
5551     PVE::Storage::deactivate_volumes($cfg, $vollist);
5552
5553     unlink $mapfifo;
5554
5555     if ($err) {
5556         rmtree $tmpdir;
5557         unlink $tmpfn;
5558
5559         foreach my $devname (keys %$devinfo) {
5560             my $volid = $devinfo->{$devname}->{volid};
5561             next if !$volid;
5562             eval {
5563                 if ($volid =~ m|^/|) {
5564                     unlink $volid || die 'unlink failed\n';
5565                 } else {
5566                     PVE::Storage::vdisk_free($cfg, $volid);
5567                 }
5568                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5569             };
5570             print STDERR "unable to cleanup '$volid' - $@" if $@;
5571         }
5572         die $err;
5573     }
5574
5575     rmtree $tmpdir;
5576
5577     rename($tmpfn, $conffile) ||
5578         die "unable to commit configuration file '$conffile'\n";
5579
5580     PVE::Cluster::cfs_update(); # make sure we read new file
5581
5582     eval { rescan($vmid, 1); };
5583     warn $@ if $@;
5584 }
5585
5586 sub restore_tar_archive {
5587     my ($archive, $vmid, $user, $opts) = @_;
5588
5589     if ($archive ne '-') {
5590         my $firstfile = tar_archive_read_firstfile($archive);
5591         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5592             if $firstfile ne 'qemu-server.conf';
5593     }
5594
5595     my $storecfg = PVE::Storage::config();
5596
5597     # destroy existing data - keep empty config
5598     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5599     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5600
5601     my $tocmd = "/usr/lib/qemu-server/qmextract";
5602
5603     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5604     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5605     $tocmd .= ' --prealloc' if $opts->{prealloc};
5606     $tocmd .= ' --info' if $opts->{info};
5607
5608     # tar option "xf" does not autodetect compression when read from STDIN,
5609     # so we pipe to zcat
5610     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5611         PVE::Tools::shellquote("--to-command=$tocmd");
5612
5613     my $tmpdir = "/var/tmp/vzdumptmp$$";
5614     mkpath $tmpdir;
5615
5616     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5617     local $ENV{VZDUMP_VMID} = $vmid;
5618     local $ENV{VZDUMP_USER} = $user;
5619
5620     my $conffile = PVE::QemuConfig->config_file($vmid);
5621     my $tmpfn = "$conffile.$$.tmp";
5622
5623     # disable interrupts (always do cleanups)
5624     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5625         print STDERR "got interrupt - ignored\n";
5626     };
5627
5628     eval {
5629         # enable interrupts
5630         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5631             die "interrupted by signal\n";
5632         };
5633
5634         if ($archive eq '-') {
5635             print "extracting archive from STDIN\n";
5636             run_command($cmd, input => "<&STDIN");
5637         } else {
5638             print "extracting archive '$archive'\n";
5639             run_command($cmd);
5640         }
5641
5642         return if $opts->{info};
5643
5644         # read new mapping
5645         my $map = {};
5646         my $statfile = "$tmpdir/qmrestore.stat";
5647         if (my $fd = IO::File->new($statfile, "r")) {
5648             while (defined (my $line = <$fd>)) {
5649                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5650                     $map->{$1} = $2 if $1;
5651                 } else {
5652                     print STDERR "unable to parse line in statfile - $line\n";
5653                 }
5654             }
5655             $fd->close();
5656         }
5657
5658         my $confsrc = "$tmpdir/qemu-server.conf";
5659
5660         my $srcfd = new IO::File($confsrc, "r") ||
5661             die "unable to open file '$confsrc'\n";
5662
5663         my $outfd = new IO::File ($tmpfn, "w") ||
5664             die "unable to write config for VM $vmid\n";
5665
5666         my $cookie = { netcount => 0 };
5667         while (defined (my $line = <$srcfd>)) {
5668             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5669         }
5670
5671         $srcfd->close();
5672         $outfd->close();
5673     };
5674     my $err = $@;
5675
5676     if ($err) {
5677
5678         unlink $tmpfn;
5679
5680         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5681
5682         die $err;
5683     }
5684
5685     rmtree $tmpdir;
5686
5687     rename $tmpfn, $conffile ||
5688         die "unable to commit configuration file '$conffile'\n";
5689
5690     PVE::Cluster::cfs_update(); # make sure we read new file
5691
5692     eval { rescan($vmid, 1); };
5693     warn $@ if $@;
5694 };
5695
5696 sub foreach_writable_storage {
5697     my ($conf, $func) = @_;
5698
5699     my $sidhash = {};
5700
5701     foreach my $ds (keys %$conf) {
5702         next if !is_valid_drivename($ds);
5703
5704         my $drive = parse_drive($ds, $conf->{$ds});
5705         next if !$drive;
5706         next if drive_is_cdrom($drive);
5707
5708         my $volid = $drive->{file};
5709
5710         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5711         $sidhash->{$sid} = $sid if $sid;
5712     }
5713
5714     foreach my $sid (sort keys %$sidhash) {
5715         &$func($sid);
5716     }
5717 }
5718
5719 sub do_snapshots_with_qemu {
5720     my ($storecfg, $volid) = @_;
5721
5722     my $storage_name = PVE::Storage::parse_volume_id($volid);
5723
5724     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5725         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5726         return 1;
5727     }
5728
5729     if ($volid =~ m/\.(qcow2|qed)$/){
5730         return 1;
5731     }
5732
5733     return undef;
5734 }
5735
5736 sub qga_check_running {
5737     my ($vmid) = @_;
5738
5739     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5740     if ($@) {
5741         warn "Qemu Guest Agent are not running - $@";
5742         return 0;
5743     }
5744     return 1;
5745 }
5746
5747 sub template_create {
5748     my ($vmid, $conf, $disk) = @_;
5749
5750     my $storecfg = PVE::Storage::config();
5751
5752     foreach_drive($conf, sub {
5753         my ($ds, $drive) = @_;
5754
5755         return if drive_is_cdrom($drive);
5756         return if $disk && $ds ne $disk;
5757
5758         my $volid = $drive->{file};
5759         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5760
5761         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5762         $drive->{file} = $voliddst;
5763         $conf->{$ds} = print_drive($vmid, $drive);
5764         PVE::QemuConfig->write_config($vmid, $conf);
5765     });
5766 }
5767
5768 sub qemu_img_convert {
5769     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5770
5771     my $storecfg = PVE::Storage::config();
5772     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5773     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5774
5775     if ($src_storeid && $dst_storeid) {
5776
5777         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5778
5779         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5780         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5781
5782         my $src_format = qemu_img_format($src_scfg, $src_volname);
5783         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5784
5785         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5786         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5787
5788         my $cmd = [];
5789         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5790         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5791         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5792         if ($is_zero_initialized) {
5793             push @$cmd, "zeroinit:$dst_path";
5794         } else {
5795             push @$cmd, $dst_path;
5796         }
5797
5798         my $parser = sub {
5799             my $line = shift;
5800             if($line =~ m/\((\S+)\/100\%\)/){
5801                 my $percent = $1;
5802                 my $transferred = int($size * $percent / 100);
5803                 my $remaining = $size - $transferred;
5804
5805                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5806             }
5807
5808         };
5809
5810         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5811         my $err = $@;
5812         die "copy failed: $err" if $err;
5813     }
5814 }
5815
5816 sub qemu_img_format {
5817     my ($scfg, $volname) = @_;
5818
5819     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5820         return $1;
5821     } else {
5822         return "raw";
5823     }
5824 }
5825
5826 sub qemu_drive_mirror {
5827     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized) = @_;
5828
5829     my $storecfg = PVE::Storage::config();
5830     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5831
5832     my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5833
5834     my $format = qemu_img_format($dst_scfg, $dst_volname);
5835
5836     my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5837
5838     my $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5839
5840     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5841     $opts->{format} = $format if $format;
5842
5843     print "drive mirror is starting (scanning bitmap) : this step can take some minutes/hours, depend of disk size and storage speed\n";
5844
5845     my $finish_job = sub {
5846         while (1) {
5847             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5848             my $stat = @$stats[0];
5849             last if !$stat;
5850             sleep 1;
5851         }
5852     };
5853
5854     eval {
5855     vm_mon_cmd($vmid, "drive-mirror", %$opts);
5856         while (1) {
5857             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5858             my $stat = @$stats[0];
5859             die "mirroring job seem to have die. Maybe do you have bad sectors?" if !$stat;
5860             die "error job is not mirroring" if $stat->{type} ne "mirror";
5861
5862             my $busy = $stat->{busy};
5863             my $ready = $stat->{ready};
5864
5865             if (my $total = $stat->{len}) {
5866                 my $transferred = $stat->{offset} || 0;
5867                 my $remaining = $total - $transferred;
5868                 my $percent = sprintf "%.2f", ($transferred * 100 / $total);
5869
5870                 print "transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
5871             }
5872
5873
5874             if ($stat->{ready} eq 'true') {
5875
5876                 last if $vmiddst != $vmid;
5877
5878                 # try to switch the disk if source and destination are on the same guest
5879                 eval { vm_mon_cmd($vmid, "block-job-complete", device => "drive-$drive") };
5880                 if (!$@) {
5881                     &$finish_job();
5882                     last;
5883                 }
5884                 die $@ if $@ !~ m/cannot be completed/;
5885             }
5886             sleep 1;
5887         }
5888
5889
5890     };
5891     my $err = $@;
5892
5893     my $cancel_job = sub {
5894         vm_mon_cmd($vmid, "block-job-cancel", device => "drive-$drive");
5895         &$finish_job();
5896     };
5897
5898     if ($err) {
5899         eval { &$cancel_job(); };
5900         die "mirroring error: $err";
5901     }
5902
5903     if ($vmiddst != $vmid) {
5904         # if we clone a disk for a new target vm, we don't switch the disk
5905         &$cancel_job(); # so we call block-job-cancel
5906     }
5907 }
5908
5909 sub clone_disk {
5910     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
5911         $newvmid, $storage, $format, $full, $newvollist) = @_;
5912
5913     my $newvolid;
5914
5915     if (!$full) {
5916         print "create linked clone of drive $drivename ($drive->{file})\n";
5917         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
5918         push @$newvollist, $newvolid;
5919     } else {
5920         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
5921         $storeid = $storage if $storage;
5922
5923         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
5924         if (!$format) {
5925             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
5926             $format = qemu_img_format($scfg, $volname);
5927         }
5928
5929         # test if requested format is supported - else use default
5930         my $supported = grep { $_ eq $format } @$validFormats;
5931         $format = $defFormat if !$supported;
5932
5933         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
5934
5935         print "create full clone of drive $drivename ($drive->{file})\n";
5936         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
5937         push @$newvollist, $newvolid;
5938
5939         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
5940
5941         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
5942         if (!$running || $snapname) {
5943             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
5944         } else {
5945
5946             my $kvmver = get_running_qemu_version ($vmid);
5947             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
5948                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
5949                     if $drive->{iothread};
5950             }
5951
5952             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit);
5953         }
5954     }
5955
5956     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
5957
5958     my $disk = $drive;
5959     $disk->{format} = undef;
5960     $disk->{file} = $newvolid;
5961     $disk->{size} = $size;
5962
5963     return $disk;
5964 }
5965
5966 # this only works if VM is running
5967 sub get_current_qemu_machine {
5968     my ($vmid) = @_;
5969
5970     my $cmd = { execute => 'query-machines', arguments => {} };
5971     my $res = vm_qmp_command($vmid, $cmd);
5972
5973     my ($current, $default);
5974     foreach my $e (@$res) {
5975         $default = $e->{name} if $e->{'is-default'};
5976         $current = $e->{name} if $e->{'is-current'};
5977     }
5978
5979     # fallback to the default machine if current is not supported by qemu
5980     return $current || $default || 'pc';
5981 }
5982
5983 sub get_running_qemu_version {
5984     my ($vmid) = @_;
5985     my $cmd = { execute => 'query-version', arguments => {} };
5986     my $res = vm_qmp_command($vmid, $cmd);
5987     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
5988 }
5989
5990 sub qemu_machine_feature_enabled {
5991     my ($machine, $kvmver, $version_major, $version_minor) = @_;
5992
5993     my $current_major;
5994     my $current_minor;
5995
5996     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
5997
5998         $current_major = $3;
5999         $current_minor = $4;
6000
6001     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6002
6003         $current_major = $1;
6004         $current_minor = $2;
6005     }
6006
6007     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6008
6009
6010 }
6011
6012 sub qemu_machine_pxe {
6013     my ($vmid, $conf, $machine) = @_;
6014
6015     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6016
6017     foreach my $opt (keys %$conf) {
6018         next if $opt !~ m/^net(\d+)$/;
6019         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6020         next if !$net;
6021         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6022         return $machine.".pxe" if $romfile =~ m/pxe/;
6023         last;
6024     }
6025
6026     return $machine;
6027 }
6028
6029 sub qemu_use_old_bios_files {
6030     my ($machine_type) = @_;
6031
6032     return if !$machine_type;
6033
6034     my $use_old_bios_files = undef;
6035
6036     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6037         $machine_type = $1;
6038         $use_old_bios_files = 1;
6039     } else {
6040         my $kvmver = kvm_user_version();
6041         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6042         # load new efi bios files on migration. So this hack is required to allow
6043         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6044         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6045         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6046     }
6047
6048     return ($use_old_bios_files, $machine_type);
6049 }
6050
6051 sub lspci {
6052
6053     my $devices = {};
6054
6055     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6056             my (undef, $id, $function) = @_;
6057             my $res = { id => $id, function => $function};
6058             push @{$devices->{$id}}, $res;
6059     });
6060
6061     return $devices;
6062 }
6063
6064 sub vm_iothreads_list {
6065     my ($vmid) = @_;
6066
6067     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6068
6069     my $iothreads = {};
6070     foreach my $iothread (@$res) {
6071         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6072     }
6073
6074     return $iothreads;
6075 }
6076
6077 sub scsihw_infos {
6078     my ($conf, $drive) = @_;
6079
6080     my $maxdev = 0;
6081
6082     if ($conf->{scsihw} && ($conf->{scsihw} =~ m/^lsi/)) {
6083         $maxdev = 7;
6084     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6085         $maxdev = 1;
6086     } else {
6087         $maxdev = 256;
6088     }
6089
6090     my $controller = int($drive->{index} / $maxdev);
6091     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6092
6093     return ($maxdev, $controller, $controller_prefix);
6094 }
6095
6096 sub add_hyperv_enlighments {
6097     my ($cpuFlags, $winversion, $machine_type, $kvmver, $nokvm, $bios, $gpu_passthrough) = @_;
6098
6099     return if $nokvm;
6100     return if $winversion < 6;
6101     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6102
6103     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6104         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6105         push @$cpuFlags , 'hv_vapic';
6106         push @$cpuFlags , 'hv_time';
6107     } else {
6108         push @$cpuFlags , 'hv_spinlocks=0xffff';
6109     }
6110
6111     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6112         push @$cpuFlags , 'hv_reset';
6113         push @$cpuFlags , 'hv_vpindex';
6114         push @$cpuFlags , 'hv_runtime';
6115     }
6116
6117     if ($winversion >= 7) {
6118         push @$cpuFlags , 'hv_relaxed';
6119         push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6120     }
6121 }
6122
6123 sub windows_version {
6124     my ($ostype) = @_;
6125
6126     return 0 if !$ostype;
6127
6128     my $winversion = 0;
6129
6130     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6131         $winversion = 5;
6132     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6133         $winversion = 6;
6134     } elsif ($ostype =~ m/^win(\d+)$/) {
6135         $winversion = $1;
6136     }
6137
6138     return $winversion;
6139 }
6140
6141 # bash completion helper
6142
6143 sub complete_backup_archives {
6144     my ($cmdname, $pname, $cvalue) = @_;
6145
6146     my $cfg = PVE::Storage::config();
6147
6148     my $storeid;
6149
6150     if ($cvalue =~ m/^([^:]+):/) {
6151         $storeid = $1;
6152     }
6153
6154     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6155
6156     my $res = [];
6157     foreach my $id (keys %$data) {
6158         foreach my $item (@{$data->{$id}}) {
6159             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6160             push @$res, $item->{volid} if defined($item->{volid});
6161         }
6162     }
6163
6164     return $res;
6165 }
6166
6167 my $complete_vmid_full = sub {
6168     my ($running) = @_;
6169
6170     my $idlist = vmstatus();
6171
6172     my $res = [];
6173
6174     foreach my $id (keys %$idlist) {
6175         my $d = $idlist->{$id};
6176         if (defined($running)) {
6177             next if $d->{template};
6178             next if $running && $d->{status} ne 'running';
6179             next if !$running && $d->{status} eq 'running';
6180         }
6181         push @$res, $id;
6182
6183     }
6184     return $res;
6185 };
6186
6187 sub complete_vmid {
6188     return &$complete_vmid_full();
6189 }
6190
6191 sub complete_vmid_stopped {
6192     return &$complete_vmid_full(0);
6193 }
6194
6195 sub complete_vmid_running {
6196     return &$complete_vmid_full(1);
6197 }
6198
6199 sub complete_storage {
6200
6201     my $cfg = PVE::Storage::config();
6202     my $ids = $cfg->{ids};
6203
6204     my $res = [];
6205     foreach my $sid (keys %$ids) {
6206         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6207         next if !$ids->{$sid}->{content}->{images};
6208         push @$res, $sid;
6209     }
6210
6211     return $res;
6212 }
6213
6214 1;