PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42
  43 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  44
  45 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  46
  47 # Note about locking: we use flock on the config file protect
  48 # against concurent actions.
  49 # Aditionaly, we have a 'lock' setting in the config file. This
  50 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  51 # allowed when such lock is set. But you can ignore this kind of
  52 # lock with the --skiplock flag.
  53
  54 cfs_register_file('/qemu-server/',
  55                   \&parse_vm_config,
  56                   \&write_vm_config);
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  72     type => 'string',
  73     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  74     description => "The drive's backing file's data format.",
  75     optional => 1,
  76 });
  77
  78 #no warnings 'redefine';
  79
  80 sub cgroups_write {
  81    my ($controller, $vmid, $option, $value) = @_;
  82
  83    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  84    PVE::ProcFSTools::write_proc_entry($path, $value);
  85
  86 }
  87
  88 my $nodename = PVE::INotify::nodename();
  89
  90 mkdir "/etc/pve/nodes/$nodename";
  91 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  92 mkdir $confdir;
  93
  94 my $var_run_tmpdir = "/var/run/qemu-server";
  95 mkdir $var_run_tmpdir;
  96
  97 my $lock_dir = "/var/lock/qemu-server";
  98 mkdir $lock_dir;
  99
 100 my $pcisysfs = "/sys/bus/pci";
 101
 102 my $cpu_vendor_list = {
 103     # Intel CPUs
 104     486 => 'GenuineIntel',
 105     pentium => 'GenuineIntel',
 106     pentium2  => 'GenuineIntel',
 107     pentium3  => 'GenuineIntel',
 108     coreduo => 'GenuineIntel',
 109     core2duo => 'GenuineIntel',
 110     Conroe  => 'GenuineIntel',
 111     Penryn  => 'GenuineIntel',
 112     Nehalem  => 'GenuineIntel',
 113     Westmere => 'GenuineIntel',
 114     SandyBridge => 'GenuineIntel',
 115     IvyBridge => 'GenuineIntel',
 116     Haswell => 'GenuineIntel',
 117     'Haswell-noTSX' => 'GenuineIntel',
 118     Broadwell => 'GenuineIntel',
 119     'Broadwell-noTSX' => 'GenuineIntel',
 120     'Skylake-Client' => 'GenuineIntel',
 121
 122     # AMD CPUs
 123     athlon => 'AuthenticAMD',
 124     phenom  => 'AuthenticAMD',
 125     Opteron_G1  => 'AuthenticAMD',
 126     Opteron_G2  => 'AuthenticAMD',
 127     Opteron_G3  => 'AuthenticAMD',
 128     Opteron_G4  => 'AuthenticAMD',
 129     Opteron_G5  => 'AuthenticAMD',
 130
 131     # generic types, use vendor from host node
 132     host => 'default',
 133     kvm32 => 'default',
 134     kvm64 => 'default',
 135     qemu32 => 'default',
 136     qemu64 => 'default',
 137 };
 138
 139 my $cpu_fmt = {
 140     cputype => {
 141         description => "Emulated CPU type.",
 142         type => 'string',
 143         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 144         default => 'kvm64',
 145         default_key => 1,
 146     },
 147     hidden => {
 148         description => "Do not identify as a KVM virtual machine.",
 149         type => 'boolean',
 150         optional => 1,
 151         default => 0
 152     },
 153 };
 154
 155 my $watchdog_fmt = {
 156     model => {
 157         default_key => 1,
 158         type => 'string',
 159         enum => [qw(i6300esb ib700)],
 160         description => "Watchdog type to emulate.",
 161         default => 'i6300esb',
 162         optional => 1,
 163     },
 164     action => {
 165         type => 'string',
 166         enum => [qw(reset shutdown poweroff pause debug none)],
 167         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 168         optional => 1,
 169     },
 170 };
 171 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 172
 173 my $confdesc = {
 174     onboot => {
 175         optional => 1,
 176         type => 'boolean',
 177         description => "Specifies whether a VM will be started during system bootup.",
 178         default => 0,
 179     },
 180     autostart => {
 181         optional => 1,
 182         type => 'boolean',
 183         description => "Automatic restart after crash (currently ignored).",
 184         default => 0,
 185     },
 186     hotplug => {
 187         optional => 1,
 188         type => 'string', format => 'pve-hotplug-features',
 189         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 190         default => 'network,disk,usb',
 191     },
 192     reboot => {
 193         optional => 1,
 194         type => 'boolean',
 195         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 196         default => 1,
 197     },
 198     lock => {
 199         optional => 1,
 200         type => 'string',
 201         description => "Lock/unlock the VM.",
 202         enum => [qw(migrate backup snapshot rollback)],
 203     },
 204     cpulimit => {
 205         optional => 1,
 206         type => 'number',
 207         description => "Limit of CPU usage.",
 208         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 209         minimum => 0,
 210         maximum => 128,
 211         default => 0,
 212     },
 213     cpuunits => {
 214         optional => 1,
 215         type => 'integer',
 216         description => "CPU weight for a VM.",
 217         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 218         minimum => 2,
 219         maximum => 262144,
 220         default => 1024,
 221     },
 222     memory => {
 223         optional => 1,
 224         type => 'integer',
 225         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 226         minimum => 16,
 227         default => 512,
 228     },
 229     balloon => {
 230         optional => 1,
 231         type => 'integer',
 232         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 233         minimum => 0,
 234     },
 235     shares => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 239         minimum => 0,
 240         maximum => 50000,
 241         default => 1000,
 242     },
 243     keyboard => {
 244         optional => 1,
 245         type => 'string',
 246         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.".
 247                        "It should not be necessary to set it.",
 248         enum => PVE::Tools::kvmkeymaplist(),
 249         default => undef,
 250     },
 251     name => {
 252         optional => 1,
 253         type => 'string', format => 'dns-name',
 254         description => "Set a name for the VM. Only used on the configuration web interface.",
 255     },
 256     scsihw => {
 257         optional => 1,
 258         type => 'string',
 259         description => "SCSI controller model",
 260         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 261         default => 'lsi',
 262     },
 263     description => {
 264         optional => 1,
 265         type => 'string',
 266         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 267     },
 268     ostype => {
 269         optional => 1,
 270         type => 'string',
 271         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 272         description => "Specify guest operating system.",
 273         verbose_description => <<EODESC,
 274 Specify guest operating system. This is used to enable special
 275 optimization/features for specific operating systems:
 276
 277 [horizontal]
 278 other;; unspecified OS
 279 wxp;; Microsoft Windows XP
 280 w2k;; Microsoft Windows 2000
 281 w2k3;; Microsoft Windows 2003
 282 w2k8;; Microsoft Windows 2008
 283 wvista;; Microsoft Windows Vista
 284 win7;; Microsoft Windows 7
 285 win8;; Microsoft Windows 8/2012/2012r2
 286 win10;; Microsoft Windows 10/2016
 287 l24;; Linux 2.4 Kernel
 288 l26;; Linux 2.6/3.X Kernel
 289 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 290 EODESC
 291     },
 292     boot => {
 293         optional => 1,
 294         type => 'string',
 295         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 296         pattern => '[acdn]{1,4}',
 297         default => 'cdn',
 298     },
 299     bootdisk => {
 300         optional => 1,
 301         type => 'string', format => 'pve-qm-bootdisk',
 302         description => "Enable booting from specified disk.",
 303         pattern => '(ide|sata|scsi|virtio)\d+',
 304     },
 305     smp => {
 306         optional => 1,
 307         type => 'integer',
 308         description => "The number of CPUs. Please use option -sockets instead.",
 309         minimum => 1,
 310         default => 1,
 311     },
 312     sockets => {
 313         optional => 1,
 314         type => 'integer',
 315         description => "The number of CPU sockets.",
 316         minimum => 1,
 317         default => 1,
 318     },
 319     cores => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "The number of cores per socket.",
 323         minimum => 1,
 324         default => 1,
 325     },
 326     numa => {
 327         optional => 1,
 328         type => 'boolean',
 329         description => "Enable/disable NUMA.",
 330         default => 0,
 331     },
 332     hugepages => {
 333         optional => 1,
 334         type => 'string',
 335         description => "Enable/disable hugepages memory.",
 336         enum => [qw(any 2 1024)],
 337     },
 338     vcpus => {
 339         optional => 1,
 340         type => 'integer',
 341         description => "Number of hotplugged vcpus.",
 342         minimum => 1,
 343         default => 0,
 344     },
 345     acpi => {
 346         optional => 1,
 347         type => 'boolean',
 348         description => "Enable/disable ACPI.",
 349         default => 1,
 350     },
 351     agent => {
 352         optional => 1,
 353         type => 'boolean',
 354         description => "Enable/disable Qemu GuestAgent.",
 355         default => 0,
 356     },
 357     kvm => {
 358         optional => 1,
 359         type => 'boolean',
 360         description => "Enable/disable KVM hardware virtualization.",
 361         default => 1,
 362     },
 363     tdf => {
 364         optional => 1,
 365         type => 'boolean',
 366         description => "Enable/disable time drift fix.",
 367         default => 0,
 368     },
 369     localtime => {
 370         optional => 1,
 371         type => 'boolean',
 372         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 373     },
 374     freeze => {
 375         optional => 1,
 376         type => 'boolean',
 377         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 378     },
 379     vga => {
 380         optional => 1,
 381         type => 'string',
 382         description => "Select the VGA type.",
 383         verbose_description => "Select the VGA type. If you want to use high resolution" .
 384             " modes (>= 1280x1024x16) then you should use the options " .
 385             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 386             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 387             "display sever. For win* OS you can select how many independent " .
 388             "displays you want, Linux guests can add displays them self. " .
 389             "You can also run without any graphic card, using a serial device" .
 390             " as terminal.",
 391         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 392     },
 393     watchdog => {
 394         optional => 1,
 395         type => 'string', format => 'pve-qm-watchdog',
 396         description => "Create a virtual hardware watchdog device.",
 397         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 398             " (by a guest action), the watchdog must be periodically polled " .
 399             "by an agent inside the guest or else the watchdog will reset " .
 400             "the guest (or execute the respective action specified)",
 401     },
 402     startdate => {
 403         optional => 1,
 404         type => 'string',
 405         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 406         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 407         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 408         default => 'now',
 409     },
 410     startup =>  get_standard_option('pve-startup-order'),
 411     template => {
 412         optional => 1,
 413         type => 'boolean',
 414         description => "Enable/disable Template.",
 415         default => 0,
 416     },
 417     args => {
 418         optional => 1,
 419         type => 'string',
 420         description => "Arbitrary arguments passed to kvm.",
 421         verbose_description => <<EODESCR,
 422 Arbitrary arguments passed to kvm, for example:
 423
 424 args: -no-reboot -no-hpet
 425
 426 NOTE: this option is for experts only.
 427 EODESCR
 428     },
 429     tablet => {
 430         optional => 1,
 431         type => 'boolean',
 432         default => 1,
 433         description => "Enable/disable the USB tablet device.",
 434         verbose_description => "Enable/disable the USB tablet device. This device is " .
 435             "usually needed to allow absolute mouse positioning with VNC. " .
 436             "Else the mouse runs out of sync with normal VNC clients. " .
 437             "If you're running lots of console-only guests on one host, " .
 438             "you may consider disabling this to save some context switches. " .
 439             "This is turned off by default if you use spice (-vga=qxl).",
 440     },
 441     migrate_speed => {
 442         optional => 1,
 443         type => 'integer',
 444         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 445         minimum => 0,
 446         default => 0,
 447     },
 448     migrate_downtime => {
 449         optional => 1,
 450         type => 'number',
 451         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 452         minimum => 0,
 453         default => 0.1,
 454     },
 455     cdrom => {
 456         optional => 1,
 457         type => 'string', format => 'pve-qm-ide',
 458         typetext => '<volume>',
 459         description => "This is an alias for option -ide2",
 460     },
 461     cpu => {
 462         optional => 1,
 463         description => "Emulated CPU type.",
 464         type => 'string',
 465         format => $cpu_fmt,
 466     },
 467     parent => get_standard_option('pve-snapshot-name', {
 468         optional => 1,
 469         description => "Parent snapshot name. This is used internally, and should not be modified.",
 470     }),
 471     snaptime => {
 472         optional => 1,
 473         description => "Timestamp for snapshots.",
 474         type => 'integer',
 475         minimum => 0,
 476     },
 477     vmstate => {
 478         optional => 1,
 479         type => 'string', format => 'pve-volume-id',
 480         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 481     },
 482     vmstatestorage => get_standard_option('pve-storage-id', {
 483         description => "Default storage for VM state volumes/files.",
 484         optional => 1,
 485     }),
 486     machine => {
 487         description => "Specific the Qemu machine type.",
 488         type => 'string',
 489         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 490         maxLength => 40,
 491         optional => 1,
 492     },
 493     smbios1 => {
 494         description => "Specify SMBIOS type 1 fields.",
 495         type => 'string', format => 'pve-qm-smbios1',
 496         maxLength => 256,
 497         optional => 1,
 498     },
 499     protection => {
 500         optional => 1,
 501         type => 'boolean',
 502         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 503         default => 0,
 504     },
 505     bios => {
 506         optional => 1,
 507         type => 'string',
 508         enum => [ qw(seabios ovmf) ],
 509         description => "Select BIOS implementation.",
 510         default => 'seabios',
 511     },
 512 };
 513
 514 # what about other qemu settings ?
 515 #cpu => 'string',
 516 #machine => 'string',
 517 #fda => 'file',
 518 #fdb => 'file',
 519 #mtdblock => 'file',
 520 #sd => 'file',
 521 #pflash => 'file',
 522 #snapshot => 'bool',
 523 #bootp => 'file',
 524 ##tftp => 'dir',
 525 ##smb => 'dir',
 526 #kernel => 'file',
 527 #append => 'string',
 528 #initrd => 'file',
 529 ##soundhw => 'string',
 530
 531 while (my ($k, $v) = each %$confdesc) {
 532     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 533 }
 534
 535 my $MAX_IDE_DISKS = 4;
 536 my $MAX_SCSI_DISKS = 14;
 537 my $MAX_VIRTIO_DISKS = 16;
 538 my $MAX_SATA_DISKS = 6;
 539 my $MAX_USB_DEVICES = 5;
 540 my $MAX_NETS = 32;
 541 my $MAX_UNUSED_DISKS = 8;
 542 my $MAX_HOSTPCI_DEVICES = 4;
 543 my $MAX_SERIAL_PORTS = 4;
 544 my $MAX_PARALLEL_PORTS = 3;
 545 my $MAX_NUMA = 8;
 546
 547 my $numa_fmt = {
 548     cpus => {
 549         type => "string",
 550         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 551         description => "CPUs accessing this NUMA node.",
 552         format_description => "id[-id];...",
 553     },
 554     memory => {
 555         type => "number",
 556         description => "Amount of memory this NUMA node provides.",
 557         optional => 1,
 558     },
 559     hostnodes => {
 560         type => "string",
 561         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 562         description => "Host NUMA nodes to use.",
 563         format_description => "id[-id];...",
 564         optional => 1,
 565     },
 566     policy => {
 567         type => 'string',
 568         enum => [qw(preferred bind interleave)],
 569         description => "NUMA allocation policy.",
 570         optional => 1,
 571     },
 572 };
 573 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 574 my $numadesc = {
 575     optional => 1,
 576     type => 'string', format => $numa_fmt,
 577     description => "NUMA topology.",
 578 };
 579 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 580
 581 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 582     $confdesc->{"numa$i"} = $numadesc;
 583 }
 584
 585 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 586                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 587                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 588 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 589
 590 my $net_fmt_bridge_descr = <<__EOD__;
 591 Bridge to attach the network device to. The Proxmox VE standard bridge
 592 is called 'vmbr0'.
 593
 594 If you do not specify a bridge, we create a kvm user (NATed) network
 595 device, which provides DHCP and DNS services. The following addresses
 596 are used:
 597
 598  10.0.2.2   Gateway
 599  10.0.2.3   DNS Server
 600  10.0.2.4   SMB Server
 601
 602 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 603 __EOD__
 604
 605 my $net_fmt = {
 606     macaddr => {
 607         type => 'string',
 608         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 609         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 610         format_description => "XX:XX:XX:XX:XX:XX",
 611         optional => 1,
 612     },
 613     model => {
 614         type => 'string',
 615         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 616         enum => $nic_model_list,
 617         default_key => 1,
 618     },
 619     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 620     bridge => {
 621         type => 'string',
 622         description => $net_fmt_bridge_descr,
 623         format_description => 'bridge',
 624         optional => 1,
 625     },
 626     queues => {
 627         type => 'integer',
 628         minimum => 0, maximum => 16,
 629         description => 'Number of packet queues to be used on the device.',
 630         optional => 1,
 631     },
 632     rate => {
 633         type => 'number',
 634         minimum => 0,
 635         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 636         optional => 1,
 637     },
 638     tag => {
 639         type => 'integer',
 640         minimum => 1, maximum => 4094,
 641         description => 'VLAN tag to apply to packets on this interface.',
 642         optional => 1,
 643     },
 644     trunks => {
 645         type => 'string',
 646         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 647         description => 'VLAN trunks to pass through this interface.',
 648         format_description => 'vlanid[;vlanid...]',
 649         optional => 1,
 650     },
 651     firewall => {
 652         type => 'boolean',
 653         description => 'Whether this interface should be protected by the firewall.',
 654         optional => 1,
 655     },
 656     link_down => {
 657         type => 'boolean',
 658         description => 'Whether this interface should be disconnected (like pulling the plug).',
 659         optional => 1,
 660     },
 661 };
 662
 663 my $netdesc = {
 664     optional => 1,
 665     type => 'string', format => $net_fmt,
 666     description => "Specify network devices.",
 667 };
 668
 669 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 670
 671 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 672     $confdesc->{"net$i"} = $netdesc;
 673 }
 674
 675 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 676 sub verify_volume_id_or_qm_path {
 677     my ($volid, $noerr) = @_;
 678
 679     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 680         return $volid;
 681     }
 682
 683     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 684     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 685     if ($@) {
 686         return undef if $noerr;
 687         die $@;
 688     }
 689     return $volid;
 690 }
 691
 692 my $drivename_hash;
 693
 694 my %drivedesc_base = (
 695     volume => { alias => 'file' },
 696     file => {
 697         type => 'string',
 698         format => 'pve-volume-id-or-qm-path',
 699         default_key => 1,
 700         format_description => 'volume',
 701         description => "The drive's backing volume.",
 702     },
 703     media => {
 704         type => 'string',
 705         enum => [qw(cdrom disk)],
 706         description => "The drive's media type.",
 707         default => 'disk',
 708         optional => 1
 709     },
 710     cyls => {
 711         type => 'integer',
 712         description => "Force the drive's physical geometry to have a specific cylinder count.",
 713         optional => 1
 714     },
 715     heads => {
 716         type => 'integer',
 717         description => "Force the drive's physical geometry to have a specific head count.",
 718         optional => 1
 719     },
 720     secs => {
 721         type => 'integer',
 722         description => "Force the drive's physical geometry to have a specific sector count.",
 723         optional => 1
 724     },
 725     trans => {
 726         type => 'string',
 727         enum => [qw(none lba auto)],
 728         description => "Force disk geometry bios translation mode.",
 729         optional => 1,
 730     },
 731     snapshot => {
 732         type => 'boolean',
 733         description => "Whether the drive should be included when making snapshots.",
 734         optional => 1,
 735     },
 736     cache => {
 737         type => 'string',
 738         enum => [qw(none writethrough writeback unsafe directsync)],
 739         description => "The drive's cache mode",
 740         optional => 1,
 741     },
 742     format => get_standard_option('pve-qm-image-format'),
 743     size => {
 744         type => 'string',
 745         format => 'disk-size',
 746         format_description => 'DiskSize',
 747         description => "Disk size. This is purely informational and has no effect.",
 748         optional => 1,
 749     },
 750     backup => {
 751         type => 'boolean',
 752         description => "Whether the drive should be included when making backups.",
 753         optional => 1,
 754     },
 755     replicate => {
 756         type => 'boolean',
 757         description => 'Whether the drive should considered for replication jobs.',
 758         optional => 1,
 759         default => 1,
 760     },
 761     rerror => {
 762         type => 'string',
 763         enum => [qw(ignore report stop)],
 764         description => 'Read error action.',
 765         optional => 1,
 766     },
 767     werror => {
 768         type => 'string',
 769         enum => [qw(enospc ignore report stop)],
 770         description => 'Write error action.',
 771         optional => 1,
 772     },
 773     aio => {
 774         type => 'string',
 775         enum => [qw(native threads)],
 776         description => 'AIO type to use.',
 777         optional => 1,
 778     },
 779     discard => {
 780         type => 'string',
 781         enum => [qw(ignore on)],
 782         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 783         optional => 1,
 784     },
 785     detect_zeroes => {
 786         type => 'boolean',
 787         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 788         optional => 1,
 789     },
 790     serial => {
 791         type => 'string',
 792         format => 'urlencoded',
 793         format_description => 'serial',
 794         maxLength => 20*3, # *3 since it's %xx url enoded
 795         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 796         optional => 1,
 797     }
 798 );
 799
 800 my %iothread_fmt = ( iothread => {
 801         type => 'boolean',
 802         description => "Whether to use iothreads for this drive",
 803         optional => 1,
 804 });
 805
 806 my %model_fmt = (
 807     model => {
 808         type => 'string',
 809         format => 'urlencoded',
 810         format_description => 'model',
 811         maxLength => 40*3, # *3 since it's %xx url enoded
 812         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 813         optional => 1,
 814     },
 815 );
 816
 817 my %queues_fmt = (
 818     queues => {
 819         type => 'integer',
 820         description => "Number of queues.",
 821         minimum => 2,
 822         optional => 1
 823     }
 824 );
 825
 826 my %scsiblock_fmt = (
 827     scsiblock => {
 828         type => 'boolean',
 829         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 830         optional => 1,
 831         default => 0,
 832     },
 833 );
 834
 835 my $add_throttle_desc = sub {
 836     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 837     my $d = {
 838         type => $type,
 839         format_description => $unit,
 840         description => "Maximum $what in $longunit.",
 841         optional => 1,
 842     };
 843     $d->{minimum} = $minimum if defined($minimum);
 844     $drivedesc_base{$key} = $d;
 845 };
 846 # throughput: (leaky bucket)
 847 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 848 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 849 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 850 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 851 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 852 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 853 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 854 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 855 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 856
 857 # pools: (pool of IO before throttling starts taking effect)
 858 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 859 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 860 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 861 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 862 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 863 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 864
 865 # burst lengths
 866 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 867 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 868 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 869 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 870 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 871 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 872
 873 # legacy support
 874 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
 875 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
 876 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
 877 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
 878
 879 my $ide_fmt = {
 880     %drivedesc_base,
 881     %model_fmt,
 882 };
 883 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 884
 885 my $idedesc = {
 886     optional => 1,
 887     type => 'string', format => $ide_fmt,
 888     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 889 };
 890 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 891
 892 my $scsi_fmt = {
 893     %drivedesc_base,
 894     %iothread_fmt,
 895     %queues_fmt,
 896     %scsiblock_fmt,
 897 };
 898 my $scsidesc = {
 899     optional => 1,
 900     type => 'string', format => $scsi_fmt,
 901     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 902 };
 903 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 904
 905 my $sata_fmt = {
 906     %drivedesc_base,
 907 };
 908 my $satadesc = {
 909     optional => 1,
 910     type => 'string', format => $sata_fmt,
 911     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 912 };
 913 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 914
 915 my $virtio_fmt = {
 916     %drivedesc_base,
 917     %iothread_fmt,
 918 };
 919 my $virtiodesc = {
 920     optional => 1,
 921     type => 'string', format => $virtio_fmt,
 922     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 923 };
 924 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 925
 926 my $alldrive_fmt = {
 927     %drivedesc_base,
 928     %iothread_fmt,
 929     %model_fmt,
 930     %queues_fmt,
 931     %scsiblock_fmt,
 932 };
 933
 934 my $efidisk_fmt = {
 935     volume => { alias => 'file' },
 936     file => {
 937         type => 'string',
 938         format => 'pve-volume-id-or-qm-path',
 939         default_key => 1,
 940         format_description => 'volume',
 941         description => "The drive's backing volume.",
 942     },
 943     format => get_standard_option('pve-qm-image-format'),
 944     size => {
 945         type => 'string',
 946         format => 'disk-size',
 947         format_description => 'DiskSize',
 948         description => "Disk size. This is purely informational and has no effect.",
 949         optional => 1,
 950     },
 951 };
 952
 953 my $efidisk_desc = {
 954     optional => 1,
 955     type => 'string', format => $efidisk_fmt,
 956     description => "Configure a Disk for storing EFI vars",
 957 };
 958
 959 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 960
 961 my $usb_fmt = {
 962     host => {
 963         default_key => 1,
 964         type => 'string', format => 'pve-qm-usb-device',
 965         format_description => 'HOSTUSBDEVICE|spice',
 966         description => <<EODESCR,
 967 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 968
 969  'bus-port(.port)*' (decimal numbers) or
 970  'vendor_id:product_id' (hexadeciaml numbers) or
 971  'spice'
 972
 973 You can use the 'lsusb -t' command to list existing usb devices.
 974
 975 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 976
 977 The value 'spice' can be used to add a usb redirection devices for spice.
 978 EODESCR
 979     },
 980     usb3 => {
 981         optional => 1,
 982         type => 'boolean',
 983         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 984         default => 0,
 985     },
 986 };
 987
 988 my $usbdesc = {
 989     optional => 1,
 990     type => 'string', format => $usb_fmt,
 991     description => "Configure an USB device (n is 0 to 4).",
 992 };
 993 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 994
 995 # NOTE: the match-groups of this regex are used in parse_hostpci
 996 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 997 my $hostpci_fmt = {
 998     host => {
 999         default_key => 1,
1000         type => 'string',
1001         pattern => qr/$PCIRE(;$PCIRE)*/,
1002         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1003         description => <<EODESCR,
1004 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1005 of PCI virtual functions of the host. HOSTPCIID syntax is:
1006
1007 'bus:dev.func' (hexadecimal numbers)
1008
1009 You can us the 'lspci' command to list existing PCI devices.
1010 EODESCR
1011     },
1012     rombar => {
1013         type => 'boolean',
1014         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1015         optional => 1,
1016         default => 1,
1017     },
1018     romfile => {
1019         type => 'string',
1020         pattern => '[^,;]+',
1021         format_description => 'string',
1022         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1023         optional => 1,
1024     },
1025     pcie => {
1026         type => 'boolean',
1027         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1028         optional => 1,
1029         default => 0,
1030     },
1031     'x-vga' => {
1032         type => 'boolean',
1033         description =>  "Enable vfio-vga device support.",
1034         optional => 1,
1035         default => 0,
1036     },
1037 };
1038 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1039
1040 my $hostpcidesc = {
1041         optional => 1,
1042         type => 'string', format => 'pve-qm-hostpci',
1043         description => "Map host PCI devices into guest.",
1044         verbose_description =>  <<EODESCR,
1045 Map host PCI devices into guest.
1046
1047 NOTE: This option allows direct access to host hardware. So it is no longer
1048 possible to migrate such machines - use with special care.
1049
1050 CAUTION: Experimental! User reported problems with this option.
1051 EODESCR
1052 };
1053 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1054
1055 my $serialdesc = {
1056         optional => 1,
1057         type => 'string',
1058         pattern => '(/dev/.+|socket)',
1059         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1060         verbose_description =>  <<EODESCR,
1061 Create a serial device inside the VM (n is 0 to 3), and pass through a
1062 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1063 host side (use 'qm terminal' to open a terminal connection).
1064
1065 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1066
1067 CAUTION: Experimental! User reported problems with this option.
1068 EODESCR
1069 };
1070
1071 my $paralleldesc= {
1072         optional => 1,
1073         type => 'string',
1074         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1075         description =>  "Map host parallel devices (n is 0 to 2).",
1076         verbose_description =>  <<EODESCR,
1077 Map host parallel devices (n is 0 to 2).
1078
1079 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1080
1081 CAUTION: Experimental! User reported problems with this option.
1082 EODESCR
1083 };
1084
1085 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1086     $confdesc->{"parallel$i"} = $paralleldesc;
1087 }
1088
1089 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1090     $confdesc->{"serial$i"} = $serialdesc;
1091 }
1092
1093 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1094     $confdesc->{"hostpci$i"} = $hostpcidesc;
1095 }
1096
1097 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1098     $drivename_hash->{"ide$i"} = 1;
1099     $confdesc->{"ide$i"} = $idedesc;
1100 }
1101
1102 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1103     $drivename_hash->{"sata$i"} = 1;
1104     $confdesc->{"sata$i"} = $satadesc;
1105 }
1106
1107 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1108     $drivename_hash->{"scsi$i"} = 1;
1109     $confdesc->{"scsi$i"} = $scsidesc ;
1110 }
1111
1112 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1113     $drivename_hash->{"virtio$i"} = 1;
1114     $confdesc->{"virtio$i"} = $virtiodesc;
1115 }
1116
1117 $drivename_hash->{efidisk0} = 1;
1118 $confdesc->{efidisk0} = $efidisk_desc;
1119
1120 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1121     $confdesc->{"usb$i"} = $usbdesc;
1122 }
1123
1124 my $unuseddesc = {
1125     optional => 1,
1126     type => 'string', format => 'pve-volume-id',
1127     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1128 };
1129
1130 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1131     $confdesc->{"unused$i"} = $unuseddesc;
1132 }
1133
1134 my $kvm_api_version = 0;
1135
1136 sub kvm_version {
1137
1138     return $kvm_api_version if $kvm_api_version;
1139
1140     my $fh = IO::File->new("</dev/kvm") ||
1141         return 0;
1142
1143     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1144         $kvm_api_version = $v;
1145     }
1146
1147     $fh->close();
1148
1149     return  $kvm_api_version;
1150 }
1151
1152 my $kvm_user_version;
1153
1154 sub kvm_user_version {
1155
1156     return $kvm_user_version if $kvm_user_version;
1157
1158     $kvm_user_version = 'unknown';
1159
1160     my $code = sub {
1161         my $line = shift;
1162         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1163             $kvm_user_version = $2;
1164         }
1165     };
1166
1167     eval { run_command("kvm -version", outfunc => $code); };
1168     warn $@ if $@;
1169
1170     return $kvm_user_version;
1171
1172 }
1173
1174 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1175
1176 sub valid_drive_names {
1177     # order is important - used to autoselect boot disk
1178     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1179             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1180             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1181             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1182             'efidisk0');
1183 }
1184
1185 sub is_valid_drivename {
1186     my $dev = shift;
1187
1188     return defined($drivename_hash->{$dev});
1189 }
1190
1191 sub option_exists {
1192     my $key = shift;
1193     return defined($confdesc->{$key});
1194 }
1195
1196 sub nic_models {
1197     return $nic_model_list;
1198 }
1199
1200 sub os_list_description {
1201
1202     return {
1203         other => 'Other',
1204         wxp => 'Windows XP',
1205         w2k => 'Windows 2000',
1206         w2k3 =>, 'Windows 2003',
1207         w2k8 => 'Windows 2008',
1208         wvista => 'Windows Vista',
1209         win7 => 'Windows 7',
1210         win8 => 'Windows 8/2012',
1211         win10 => 'Windows 10/2016',
1212         l24 => 'Linux 2.4',
1213         l26 => 'Linux 2.6',
1214     };
1215 }
1216
1217 my $cdrom_path;
1218
1219 sub get_cdrom_path {
1220
1221     return  $cdrom_path if $cdrom_path;
1222
1223     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1224     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1225     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1226 }
1227
1228 sub get_iso_path {
1229     my ($storecfg, $vmid, $cdrom) = @_;
1230
1231     if ($cdrom eq 'cdrom') {
1232         return get_cdrom_path();
1233     } elsif ($cdrom eq 'none') {
1234         return '';
1235     } elsif ($cdrom =~ m|^/|) {
1236         return $cdrom;
1237     } else {
1238         return PVE::Storage::path($storecfg, $cdrom);
1239     }
1240 }
1241
1242 # try to convert old style file names to volume IDs
1243 sub filename_to_volume_id {
1244     my ($vmid, $file, $media) = @_;
1245
1246     if (!($file eq 'none' || $file eq 'cdrom' ||
1247           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1248
1249         return undef if $file =~ m|/|;
1250
1251         if ($media && $media eq 'cdrom') {
1252             $file = "local:iso/$file";
1253         } else {
1254             $file = "local:$vmid/$file";
1255         }
1256     }
1257
1258     return $file;
1259 }
1260
1261 sub verify_media_type {
1262     my ($opt, $vtype, $media) = @_;
1263
1264     return if !$media;
1265
1266     my $etype;
1267     if ($media eq 'disk') {
1268         $etype = 'images';
1269     } elsif ($media eq 'cdrom') {
1270         $etype = 'iso';
1271     } else {
1272         die "internal error";
1273     }
1274
1275     return if ($vtype eq $etype);
1276
1277     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1278 }
1279
1280 sub cleanup_drive_path {
1281     my ($opt, $storecfg, $drive) = @_;
1282
1283     # try to convert filesystem paths to volume IDs
1284
1285     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1286         ($drive->{file} !~ m|^/dev/.+|) &&
1287         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1288         ($drive->{file} !~ m/^\d+$/)) {
1289         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1290         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1291         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1292         verify_media_type($opt, $vtype, $drive->{media});
1293         $drive->{file} = $volid;
1294     }
1295
1296     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1297 }
1298
1299 sub parse_hotplug_features {
1300     my ($data) = @_;
1301
1302     my $res = {};
1303
1304     return $res if $data eq '0';
1305
1306     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1307
1308     foreach my $feature (PVE::Tools::split_list($data)) {
1309         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1310             $res->{$1} = 1;
1311         } else {
1312             die "invalid hotplug feature '$feature'\n";
1313         }
1314     }
1315     return $res;
1316 }
1317
1318 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1319 sub pve_verify_hotplug_features {
1320     my ($value, $noerr) = @_;
1321
1322     return $value if parse_hotplug_features($value);
1323
1324     return undef if $noerr;
1325
1326     die "unable to parse hotplug option\n";
1327 }
1328
1329 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1330 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1331 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1332 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1333 #        [,iothread=on][,serial=serial][,model=model]
1334
1335 sub parse_drive {
1336     my ($key, $data) = @_;
1337
1338     my ($interface, $index);
1339
1340     if ($key =~ m/^([^\d]+)(\d+)$/) {
1341         $interface = $1;
1342         $index = $2;
1343     } else {
1344         return undef;
1345     }
1346
1347     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1348                                      : $confdesc->{$key}->{format};
1349     if (!$desc) {
1350         warn "invalid drive key: $key\n";
1351         return undef;
1352     }
1353     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1354     return undef if !$res;
1355     $res->{interface} = $interface;
1356     $res->{index} = $index;
1357
1358     my $error = 0;
1359     foreach my $opt (qw(bps bps_rd bps_wr)) {
1360         if (my $bps = defined(delete $res->{$opt})) {
1361             if (defined($res->{"m$opt"})) {
1362                 warn "both $opt and m$opt specified\n";
1363                 ++$error;
1364                 next;
1365             }
1366             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1367         }
1368     }
1369
1370     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1371     for my $requirement (
1372         [mbps_max => 'mbps'],
1373         [mbps_rd_max => 'mbps_rd'],
1374         [mbps_wr_max => 'mbps_wr'],
1375         [miops_max => 'miops'],
1376         [miops_rd_max => 'miops_rd'],
1377         [miops_wr_max => 'miops_wr'],
1378         [bps_max_length => 'mbps_max'],
1379         [bps_rd_max_length => 'mbps_rd_max'],
1380         [bps_wr_max_length => 'mbps_wr_max'],
1381         [iops_max_length => 'iops_max'],
1382         [iops_rd_max_length => 'iops_rd_max'],
1383         [iops_wr_max_length => 'iops_wr_max']) {
1384         my ($option, $requires) = @$requirement;
1385         if ($res->{$option} && !$res->{$requires}) {
1386             warn "$option requires $requires\n";
1387             ++$error;
1388         }
1389     }
1390
1391     return undef if $error;
1392
1393     return undef if $res->{mbps_rd} && $res->{mbps};
1394     return undef if $res->{mbps_wr} && $res->{mbps};
1395     return undef if $res->{iops_rd} && $res->{iops};
1396     return undef if $res->{iops_wr} && $res->{iops};
1397
1398     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1399         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1400         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1401         return undef if $res->{interface} eq 'virtio';
1402     }
1403
1404     if (my $size = $res->{size}) {
1405         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1406     }
1407
1408     return $res;
1409 }
1410
1411 sub print_drive {
1412     my ($vmid, $drive) = @_;
1413     my $data = { %$drive };
1414     delete $data->{$_} for qw(index interface);
1415     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1416 }
1417
1418 sub scsi_inquiry {
1419     my($fh, $noerr) = @_;
1420
1421     my $SG_IO = 0x2285;
1422     my $SG_GET_VERSION_NUM = 0x2282;
1423
1424     my $versionbuf = "\x00" x 8;
1425     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1426     if (!$ret) {
1427         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1428         return undef;
1429     }
1430     my $version = unpack("I", $versionbuf);
1431     if ($version < 30000) {
1432         die "scsi generic interface too old\n"  if !$noerr;
1433         return undef;
1434     }
1435
1436     my $buf = "\x00" x 36;
1437     my $sensebuf = "\x00" x 8;
1438     my $cmd = pack("C x3 C x1", 0x12, 36);
1439
1440     # see /usr/include/scsi/sg.h
1441     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1442
1443     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1444                       length($sensebuf), 0, length($buf), $buf,
1445                       $cmd, $sensebuf, 6000);
1446
1447     $ret = ioctl($fh, $SG_IO, $packet);
1448     if (!$ret) {
1449         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1450         return undef;
1451     }
1452
1453     my @res = unpack($sg_io_hdr_t, $packet);
1454     if ($res[17] || $res[18]) {
1455         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1456         return undef;
1457     }
1458
1459     my $res = {};
1460     (my $byte0, my $byte1, $res->{vendor},
1461      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1462
1463     $res->{removable} = $byte1 & 128 ? 1 : 0;
1464     $res->{type} = $byte0 & 31;
1465
1466     return $res;
1467 }
1468
1469 sub path_is_scsi {
1470     my ($path) = @_;
1471
1472     my $fh = IO::File->new("+<$path") || return undef;
1473     my $res = scsi_inquiry($fh, 1);
1474     close($fh);
1475
1476     return $res;
1477 }
1478
1479 sub machine_type_is_q35 {
1480     my ($conf) = @_;
1481
1482     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1483 }
1484
1485 sub print_tabletdevice_full {
1486     my ($conf) = @_;
1487
1488     my $q35 = machine_type_is_q35($conf);
1489
1490     # we use uhci for old VMs because tablet driver was buggy in older qemu
1491     my $usbbus = $q35 ? "ehci" : "uhci";
1492
1493     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1494 }
1495
1496 sub print_drivedevice_full {
1497     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1498
1499     my $device = '';
1500     my $maxdev = 0;
1501
1502     if ($drive->{interface} eq 'virtio') {
1503         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1504         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1505         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1506     } elsif ($drive->{interface} eq 'scsi') {
1507
1508         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1509         my $unit = $drive->{index} % $maxdev;
1510         my $devicetype = 'hd';
1511         my $path = '';
1512         if (drive_is_cdrom($drive)) {
1513             $devicetype = 'cd';
1514         } else {
1515             if ($drive->{file} =~ m|^/|) {
1516                 $path = $drive->{file};
1517                 if (my $info = path_is_scsi($path)) {
1518                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1519                         $devicetype = 'block';
1520                     } elsif ($info->{type} == 1) { # tape
1521                         $devicetype = 'generic';
1522                     }
1523                 }
1524             } else {
1525                  $path = PVE::Storage::path($storecfg, $drive->{file});
1526             }
1527
1528             if($path =~ m/^iscsi\:\/\//){
1529                 $devicetype = 'generic';
1530             }
1531         }
1532
1533         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1534             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1535         } else {
1536             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1537         }
1538
1539     } elsif ($drive->{interface} eq 'ide'){
1540         $maxdev = 2;
1541         my $controller = int($drive->{index} / $maxdev);
1542         my $unit = $drive->{index} % $maxdev;
1543         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1544
1545         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1546         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1547             $model = URI::Escape::uri_unescape($model);
1548             $device .= ",model=$model";
1549         }
1550     } elsif ($drive->{interface} eq 'sata'){
1551         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1552         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1553         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1554     } elsif ($drive->{interface} eq 'usb') {
1555         die "implement me";
1556         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1557     } else {
1558         die "unsupported interface type";
1559     }
1560
1561     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1562
1563     return $device;
1564 }
1565
1566 sub get_initiator_name {
1567     my $initiator;
1568
1569     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1570     while (defined(my $line = <$fh>)) {
1571         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1572         $initiator = $1;
1573         last;
1574     }
1575     $fh->close();
1576
1577     return $initiator;
1578 }
1579
1580 sub print_drive_full {
1581     my ($storecfg, $vmid, $drive) = @_;
1582
1583     my $path;
1584     my $volid = $drive->{file};
1585     my $format;
1586
1587     if (drive_is_cdrom($drive)) {
1588         $path = get_iso_path($storecfg, $vmid, $volid);
1589     } else {
1590         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1591         if ($storeid) {
1592             $path = PVE::Storage::path($storecfg, $volid);
1593             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1594             $format = qemu_img_format($scfg, $volname);
1595         } else {
1596             $path = $volid;
1597             $format = "raw";
1598         }
1599    }
1600
1601     my $opts = '';
1602     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard);
1603     foreach my $o (@qemu_drive_options) {
1604         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1605     }
1606     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1607         my ($dir, $qmpname) = @$type;
1608         if (my $v = $drive->{"mbps$dir"}) {
1609             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1610         }
1611         if (my $v = $drive->{"mbps${dir}_max"}) {
1612             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1613         }
1614         if (my $v = $drive->{"bps${dir}_max_length"}) {
1615             $opts .= ",throttling.bps$qmpname-max-length=$v";
1616         }
1617         if (my $v = $drive->{"iops${dir}"}) {
1618             $opts .= ",throttling.iops$qmpname=$v";
1619         }
1620         if (my $v = $drive->{"iops${dir}_max"}) {
1621             $opts .= ",throttling.iops$qmpname-max=$v";
1622         }
1623         if (my $v = $drive->{"iops${dir}_max_length"}) {
1624             $opts .= ",throttling.iops$qmpname-max-length=$v";
1625         }
1626     }
1627
1628     if (my $serial = $drive->{serial}) {
1629         $serial = URI::Escape::uri_unescape($serial);
1630         $opts .= ",serial=$serial";
1631     }
1632
1633     $opts .= ",format=$format" if $format && !$drive->{format};
1634
1635     my $cache_direct = 0;
1636
1637     if (my $cache = $drive->{cache}) {
1638         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1639     } elsif (!drive_is_cdrom($drive)) {
1640         $opts .= ",cache=none";
1641         $cache_direct = 1;
1642     }
1643
1644     # aio native works only with O_DIRECT
1645     if (!$drive->{aio}) {
1646         if($cache_direct) {
1647             $opts .= ",aio=native";
1648         } else {
1649             $opts .= ",aio=threads";
1650         }
1651     }
1652
1653     if (!drive_is_cdrom($drive)) {
1654         my $detectzeroes;
1655         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1656             $detectzeroes = 'off';
1657         } elsif ($drive->{discard}) {
1658             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1659         } else {
1660             # This used to be our default with discard not being specified:
1661             $detectzeroes = 'on';
1662         }
1663         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1664     }
1665
1666     my $pathinfo = $path ? "file=$path," : '';
1667
1668     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1669 }
1670
1671 sub print_netdevice_full {
1672     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1673
1674     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1675
1676     my $device = $net->{model};
1677     if ($net->{model} eq 'virtio') {
1678          $device = 'virtio-net-pci';
1679      };
1680
1681     my $pciaddr = print_pci_addr("$netid", $bridges);
1682     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1683     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1684         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1685         my $vectors = $net->{queues} * 2 + 2;
1686         $tmpstr .= ",vectors=$vectors,mq=on";
1687     }
1688     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1689
1690     if ($use_old_bios_files) {
1691         my $romfile;
1692         if ($device eq 'virtio-net-pci') {
1693             $romfile = 'pxe-virtio.rom';
1694         } elsif ($device eq 'e1000') {
1695             $romfile = 'pxe-e1000.rom';
1696         } elsif ($device eq 'ne2k') {
1697             $romfile = 'pxe-ne2k_pci.rom';
1698         } elsif ($device eq 'pcnet') {
1699             $romfile = 'pxe-pcnet.rom';
1700         } elsif ($device eq 'rtl8139') {
1701             $romfile = 'pxe-rtl8139.rom';
1702         }
1703         $tmpstr .= ",romfile=$romfile" if $romfile;
1704     }
1705
1706     return $tmpstr;
1707 }
1708
1709 sub print_netdev_full {
1710     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1711
1712     my $i = '';
1713     if ($netid =~ m/^net(\d+)$/) {
1714         $i = int($1);
1715     }
1716
1717     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1718
1719     my $ifname = "tap${vmid}i$i";
1720
1721     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1722     die "interface name '$ifname' is too long (max 15 character)\n"
1723         if length($ifname) >= 16;
1724
1725     my $vhostparam = '';
1726     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1727
1728     my $vmname = $conf->{name} || "vm$vmid";
1729
1730     my $netdev = "";
1731     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1732
1733     if ($net->{bridge}) {
1734         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1735     } else {
1736         $netdev = "type=user,id=$netid,hostname=$vmname";
1737     }
1738
1739     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1740
1741     return $netdev;
1742 }
1743
1744
1745 sub print_cpu_device {
1746     my ($conf, $id) = @_;
1747
1748     my $kvm = $conf->{kvm} // 1;
1749     my $cpu = $kvm ? "kvm64" : "qemu64";
1750     if (my $cputype = $conf->{cpu}) {
1751         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1752             or die "Cannot parse cpu description: $cputype\n";
1753         $cpu = $cpuconf->{cputype};
1754     }
1755
1756     my $cores = $conf->{cores} || 1;
1757
1758     my $current_core = ($id - 1) % $cores;
1759     my $current_socket = int(($id - 1 - $current_core)/$cores);
1760
1761     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1762 }
1763
1764 sub drive_is_cdrom {
1765     my ($drive) = @_;
1766
1767     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1768
1769 }
1770
1771 sub parse_number_sets {
1772     my ($set) = @_;
1773     my $res = [];
1774     foreach my $part (split(/;/, $set)) {
1775         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1776             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1777             push @$res, [ $1, $2 ];
1778         } else {
1779             die "invalid range: $part\n";
1780         }
1781     }
1782     return $res;
1783 }
1784
1785 sub parse_numa {
1786     my ($data) = @_;
1787
1788     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1789     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1790     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1791     return $res;
1792 }
1793
1794 sub parse_hostpci {
1795     my ($value) = @_;
1796
1797     return undef if !$value;
1798
1799     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1800
1801     my @idlist = split(/;/, $res->{host});
1802     delete $res->{host};
1803     foreach my $id (@idlist) {
1804         if ($id =~ /^$PCIRE$/) {
1805             if (defined($2)) {
1806                 push @{$res->{pciid}}, { id => $1, function => $2 };
1807             } else {
1808                 my $pcidevices = lspci($1);
1809                 $res->{pciid} = $pcidevices->{$1};
1810             }
1811         } else {
1812             # should have been caught by parse_property_string already
1813             die "failed to parse PCI id: $id\n";
1814         }
1815     }
1816     return $res;
1817 }
1818
1819 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1820 sub parse_net {
1821     my ($data) = @_;
1822
1823     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1824     if ($@) {
1825         warn $@;
1826         return undef;
1827     }
1828     if (!defined($res->{macaddr})) {
1829         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1830         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1831     }
1832     return $res;
1833 }
1834
1835 sub print_net {
1836     my $net = shift;
1837
1838     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1839 }
1840
1841 sub add_random_macs {
1842     my ($settings) = @_;
1843
1844     foreach my $opt (keys %$settings) {
1845         next if $opt !~ m/^net(\d+)$/;
1846         my $net = parse_net($settings->{$opt});
1847         next if !$net;
1848         $settings->{$opt} = print_net($net);
1849     }
1850 }
1851
1852 sub vm_is_volid_owner {
1853     my ($storecfg, $vmid, $volid) = @_;
1854
1855     if ($volid !~  m|^/|) {
1856         my ($path, $owner);
1857         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1858         if ($owner && ($owner == $vmid)) {
1859             return 1;
1860         }
1861     }
1862
1863     return undef;
1864 }
1865
1866 sub split_flagged_list {
1867     my $text = shift || '';
1868     $text =~ s/[,;]/ /g;
1869     $text =~ s/^\s+//;
1870     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1871 }
1872
1873 sub join_flagged_list {
1874     my ($how, $lst) = @_;
1875     join $how, map { $lst->{$_} . $_ } keys %$lst;
1876 }
1877
1878 sub vmconfig_delete_pending_option {
1879     my ($conf, $key, $force) = @_;
1880
1881     delete $conf->{pending}->{$key};
1882     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1883     $pending_delete_hash->{$key} = $force ? '!' : '';
1884     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1885 }
1886
1887 sub vmconfig_undelete_pending_option {
1888     my ($conf, $key) = @_;
1889
1890     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1891     delete $pending_delete_hash->{$key};
1892
1893     if (%$pending_delete_hash) {
1894         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1895     } else {
1896         delete $conf->{pending}->{delete};
1897     }
1898 }
1899
1900 sub vmconfig_register_unused_drive {
1901     my ($storecfg, $vmid, $conf, $drive) = @_;
1902
1903     if (!drive_is_cdrom($drive)) {
1904         my $volid = $drive->{file};
1905         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1906             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1907         }
1908     }
1909 }
1910
1911 sub vmconfig_cleanup_pending {
1912     my ($conf) = @_;
1913
1914     # remove pending changes when nothing changed
1915     my $changes;
1916     foreach my $opt (keys %{$conf->{pending}}) {
1917         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1918             $changes = 1;
1919             delete $conf->{pending}->{$opt};
1920         }
1921     }
1922
1923     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1924     my $pending_delete_hash = {};
1925     while (my ($opt, $force) = each %$current_delete_hash) {
1926         if (defined($conf->{$opt})) {
1927             $pending_delete_hash->{$opt} = $force;
1928         } else {
1929             $changes = 1;
1930         }
1931     }
1932
1933     if (%$pending_delete_hash) {
1934         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1935     } else {
1936         delete $conf->{pending}->{delete};
1937     }
1938
1939     return $changes;
1940 }
1941
1942 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1943 my $smbios1_fmt = {
1944     uuid => {
1945         type => 'string',
1946         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1947         format_description => 'UUID',
1948         description => "Set SMBIOS1 UUID.",
1949         optional => 1,
1950     },
1951     version => {
1952         type => 'string',
1953         pattern => '\S+',
1954         format_description => 'string',
1955         description => "Set SMBIOS1 version.",
1956         optional => 1,
1957     },
1958     serial => {
1959         type => 'string',
1960         pattern => '\S+',
1961         format_description => 'string',
1962         description => "Set SMBIOS1 serial number.",
1963         optional => 1,
1964     },
1965     manufacturer => {
1966         type => 'string',
1967         pattern => '\S+',
1968         format_description => 'string',
1969         description => "Set SMBIOS1 manufacturer.",
1970         optional => 1,
1971     },
1972     product => {
1973         type => 'string',
1974         pattern => '\S+',
1975         format_description => 'string',
1976         description => "Set SMBIOS1 product ID.",
1977         optional => 1,
1978     },
1979     sku => {
1980         type => 'string',
1981         pattern => '\S+',
1982         format_description => 'string',
1983         description => "Set SMBIOS1 SKU string.",
1984         optional => 1,
1985     },
1986     family => {
1987         type => 'string',
1988         pattern => '\S+',
1989         format_description => 'string',
1990         description => "Set SMBIOS1 family string.",
1991         optional => 1,
1992     },
1993 };
1994
1995 sub parse_smbios1 {
1996     my ($data) = @_;
1997
1998     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1999     warn $@ if $@;
2000     return $res;
2001 }
2002
2003 sub print_smbios1 {
2004     my ($smbios1) = @_;
2005     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2006 }
2007
2008 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2009
2010 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2011 sub verify_bootdisk {
2012     my ($value, $noerr) = @_;
2013
2014     return $value if is_valid_drivename($value);
2015
2016     return undef if $noerr;
2017
2018     die "invalid boot disk '$value'\n";
2019 }
2020
2021 sub parse_watchdog {
2022     my ($value) = @_;
2023
2024     return undef if !$value;
2025
2026     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2027     warn $@ if $@;
2028     return $res;
2029 }
2030
2031 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2032 sub verify_usb_device {
2033     my ($value, $noerr) = @_;
2034
2035     return $value if parse_usb_device($value);
2036
2037     return undef if $noerr;
2038
2039     die "unable to parse usb device\n";
2040 }
2041
2042 # add JSON properties for create and set function
2043 sub json_config_properties {
2044     my $prop = shift;
2045
2046     foreach my $opt (keys %$confdesc) {
2047         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2048         $prop->{$opt} = $confdesc->{$opt};
2049     }
2050
2051     return $prop;
2052 }
2053
2054 sub check_type {
2055     my ($key, $value) = @_;
2056
2057     die "unknown setting '$key'\n" if !$confdesc->{$key};
2058
2059     my $type = $confdesc->{$key}->{type};
2060
2061     if (!defined($value)) {
2062         die "got undefined value\n";
2063     }
2064
2065     if ($value =~ m/[\n\r]/) {
2066         die "property contains a line feed\n";
2067     }
2068
2069     if ($type eq 'boolean') {
2070         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2071         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2072         die "type check ('boolean') failed - got '$value'\n";
2073     } elsif ($type eq 'integer') {
2074         return int($1) if $value =~ m/^(\d+)$/;
2075         die "type check ('integer') failed - got '$value'\n";
2076     } elsif ($type eq 'number') {
2077         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2078         die "type check ('number') failed - got '$value'\n";
2079     } elsif ($type eq 'string') {
2080         if (my $fmt = $confdesc->{$key}->{format}) {
2081             PVE::JSONSchema::check_format($fmt, $value);
2082             return $value;
2083         }
2084         $value =~ s/^\"(.*)\"$/$1/;
2085         return $value;
2086     } else {
2087         die "internal error"
2088     }
2089 }
2090
2091 sub check_iommu_support{
2092     #fixme : need to check IOMMU support
2093     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2094
2095     my $iommu=1;
2096     return $iommu;
2097
2098 }
2099
2100 sub touch_config {
2101     my ($vmid) = @_;
2102
2103     my $conf = PVE::QemuConfig->config_file($vmid);
2104     utime undef, undef, $conf;
2105 }
2106
2107 sub destroy_vm {
2108     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2109
2110     my $conffile = PVE::QemuConfig->config_file($vmid);
2111
2112     my $conf = PVE::QemuConfig->load_config($vmid);
2113
2114     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2115
2116     if ($conf->{template}) {
2117         # check if any base image is still used by a linked clone
2118         foreach_drive($conf, sub {
2119                 my ($ds, $drive) = @_;
2120
2121                 return if drive_is_cdrom($drive);
2122
2123                 my $volid = $drive->{file};
2124
2125                 return if !$volid || $volid =~ m|^/|;
2126
2127                 die "base volume '$volid' is still in use by linked cloned\n"
2128                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2129
2130         });
2131     }
2132
2133     # only remove disks owned by this VM
2134     foreach_drive($conf, sub {
2135         my ($ds, $drive) = @_;
2136
2137         return if drive_is_cdrom($drive);
2138
2139         my $volid = $drive->{file};
2140
2141         return if !$volid || $volid =~ m|^/|;
2142
2143         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2144         return if !$path || !$owner || ($owner != $vmid);
2145
2146         eval {
2147             PVE::Storage::vdisk_free($storecfg, $volid);
2148         };
2149         warn "Could not remove disk '$volid', check manually: $@" if $@;
2150
2151     });
2152
2153     if ($keep_empty_config) {
2154         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2155     } else {
2156         unlink $conffile;
2157     }
2158
2159     # also remove unused disk
2160     eval {
2161         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2162
2163         eval {
2164             PVE::Storage::foreach_volid($dl, sub {
2165                 my ($volid, $sid, $volname, $d) = @_;
2166                 PVE::Storage::vdisk_free($storecfg, $volid);
2167             });
2168         };
2169         warn $@ if $@;
2170
2171     };
2172     warn $@ if $@;
2173 }
2174
2175 sub parse_vm_config {
2176     my ($filename, $raw) = @_;
2177
2178     return undef if !defined($raw);
2179
2180     my $res = {
2181         digest => Digest::SHA::sha1_hex($raw),
2182         snapshots => {},
2183         pending => {},
2184     };
2185
2186     $filename =~ m|/qemu-server/(\d+)\.conf$|
2187         || die "got strange filename '$filename'";
2188
2189     my $vmid = $1;
2190
2191     my $conf = $res;
2192     my $descr;
2193     my $section = '';
2194
2195     my @lines = split(/\n/, $raw);
2196     foreach my $line (@lines) {
2197         next if $line =~ m/^\s*$/;
2198
2199         if ($line =~ m/^\[PENDING\]\s*$/i) {
2200             $section = 'pending';
2201             if (defined($descr)) {
2202                 $descr =~ s/\s+$//;
2203                 $conf->{description} = $descr;
2204             }
2205             $descr = undef;
2206             $conf = $res->{$section} = {};
2207             next;
2208
2209         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2210             $section = $1;
2211             if (defined($descr)) {
2212                 $descr =~ s/\s+$//;
2213                 $conf->{description} = $descr;
2214             }
2215             $descr = undef;
2216             $conf = $res->{snapshots}->{$section} = {};
2217             next;
2218         }
2219
2220         if ($line =~ m/^\#(.*)\s*$/) {
2221             $descr = '' if !defined($descr);
2222             $descr .= PVE::Tools::decode_text($1) . "\n";
2223             next;
2224         }
2225
2226         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2227             $descr = '' if !defined($descr);
2228             $descr .= PVE::Tools::decode_text($2);
2229         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2230             $conf->{snapstate} = $1;
2231         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2232             my $key = $1;
2233             my $value = $2;
2234             $conf->{$key} = $value;
2235         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2236             my $value = $1;
2237             if ($section eq 'pending') {
2238                 $conf->{delete} = $value; # we parse this later
2239             } else {
2240                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2241             }
2242         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2243             my $key = $1;
2244             my $value = $2;
2245             eval { $value = check_type($key, $value); };
2246             if ($@) {
2247                 warn "vm $vmid - unable to parse value of '$key' - $@";
2248             } else {
2249                 $key = 'ide2' if $key eq 'cdrom';
2250                 my $fmt = $confdesc->{$key}->{format};
2251                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2252                     my $v = parse_drive($key, $value);
2253                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2254                         $v->{file} = $volid;
2255                         $value = print_drive($vmid, $v);
2256                     } else {
2257                         warn "vm $vmid - unable to parse value of '$key'\n";
2258                         next;
2259                     }
2260                 }
2261
2262                 $conf->{$key} = $value;
2263             }
2264         }
2265     }
2266
2267     if (defined($descr)) {
2268         $descr =~ s/\s+$//;
2269         $conf->{description} = $descr;
2270     }
2271     delete $res->{snapstate}; # just to be sure
2272
2273     return $res;
2274 }
2275
2276 sub write_vm_config {
2277     my ($filename, $conf) = @_;
2278
2279     delete $conf->{snapstate}; # just to be sure
2280
2281     if ($conf->{cdrom}) {
2282         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2283         $conf->{ide2} = $conf->{cdrom};
2284         delete $conf->{cdrom};
2285     }
2286
2287     # we do not use 'smp' any longer
2288     if ($conf->{sockets}) {
2289         delete $conf->{smp};
2290     } elsif ($conf->{smp}) {
2291         $conf->{sockets} = $conf->{smp};
2292         delete $conf->{cores};
2293         delete $conf->{smp};
2294     }
2295
2296     my $used_volids = {};
2297
2298     my $cleanup_config = sub {
2299         my ($cref, $pending, $snapname) = @_;
2300
2301         foreach my $key (keys %$cref) {
2302             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2303                 $key eq 'snapstate' || $key eq 'pending';
2304             my $value = $cref->{$key};
2305             if ($key eq 'delete') {
2306                 die "propertry 'delete' is only allowed in [PENDING]\n"
2307                     if !$pending;
2308                 # fixme: check syntax?
2309                 next;
2310             }
2311             eval { $value = check_type($key, $value); };
2312             die "unable to parse value of '$key' - $@" if $@;
2313
2314             $cref->{$key} = $value;
2315
2316             if (!$snapname && is_valid_drivename($key)) {
2317                 my $drive = parse_drive($key, $value);
2318                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2319             }
2320         }
2321     };
2322
2323     &$cleanup_config($conf);
2324
2325     &$cleanup_config($conf->{pending}, 1);
2326
2327     foreach my $snapname (keys %{$conf->{snapshots}}) {
2328         die "internal error" if $snapname eq 'pending';
2329         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2330     }
2331
2332     # remove 'unusedX' settings if we re-add a volume
2333     foreach my $key (keys %$conf) {
2334         my $value = $conf->{$key};
2335         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2336             delete $conf->{$key};
2337         }
2338     }
2339
2340     my $generate_raw_config = sub {
2341         my ($conf, $pending) = @_;
2342
2343         my $raw = '';
2344
2345         # add description as comment to top of file
2346         if (defined(my $descr = $conf->{description})) {
2347             if ($descr) {
2348                 foreach my $cl (split(/\n/, $descr)) {
2349                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2350                 }
2351             } else {
2352                 $raw .= "#\n" if $pending;
2353             }
2354         }
2355
2356         foreach my $key (sort keys %$conf) {
2357             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2358             $raw .= "$key: $conf->{$key}\n";
2359         }
2360         return $raw;
2361     };
2362
2363     my $raw = &$generate_raw_config($conf);
2364
2365     if (scalar(keys %{$conf->{pending}})){
2366         $raw .= "\n[PENDING]\n";
2367         $raw .= &$generate_raw_config($conf->{pending}, 1);
2368     }
2369
2370     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2371         $raw .= "\n[$snapname]\n";
2372         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2373     }
2374
2375     return $raw;
2376 }
2377
2378 sub load_defaults {
2379
2380     my $res = {};
2381
2382     # we use static defaults from our JSON schema configuration
2383     foreach my $key (keys %$confdesc) {
2384         if (defined(my $default = $confdesc->{$key}->{default})) {
2385             $res->{$key} = $default;
2386         }
2387     }
2388
2389     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2390     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2391
2392     return $res;
2393 }
2394
2395 sub config_list {
2396     my $vmlist = PVE::Cluster::get_vmlist();
2397     my $res = {};
2398     return $res if !$vmlist || !$vmlist->{ids};
2399     my $ids = $vmlist->{ids};
2400
2401     foreach my $vmid (keys %$ids) {
2402         my $d = $ids->{$vmid};
2403         next if !$d->{node} || $d->{node} ne $nodename;
2404         next if !$d->{type} || $d->{type} ne 'qemu';
2405         $res->{$vmid}->{exists} = 1;
2406     }
2407     return $res;
2408 }
2409
2410 # test if VM uses local resources (to prevent migration)
2411 sub check_local_resources {
2412     my ($conf, $noerr) = @_;
2413
2414     my $loc_res = 0;
2415
2416     $loc_res = 1 if $conf->{hostusb}; # old syntax
2417     $loc_res = 1 if $conf->{hostpci}; # old syntax
2418
2419     foreach my $k (keys %$conf) {
2420         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2421         # sockets are safe: they will recreated be on the target side post-migrate
2422         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2423         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2424     }
2425
2426     die "VM uses local resources\n" if $loc_res && !$noerr;
2427
2428     return $loc_res;
2429 }
2430
2431 # check if used storages are available on all nodes (use by migrate)
2432 sub check_storage_availability {
2433     my ($storecfg, $conf, $node) = @_;
2434
2435     foreach_drive($conf, sub {
2436         my ($ds, $drive) = @_;
2437
2438         my $volid = $drive->{file};
2439         return if !$volid;
2440
2441         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2442         return if !$sid;
2443
2444         # check if storage is available on both nodes
2445         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2446         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2447    });
2448 }
2449
2450 # list nodes where all VM images are available (used by has_feature API)
2451 sub shared_nodes {
2452     my ($conf, $storecfg) = @_;
2453
2454     my $nodelist = PVE::Cluster::get_nodelist();
2455     my $nodehash = { map { $_ => 1 } @$nodelist };
2456     my $nodename = PVE::INotify::nodename();
2457
2458     foreach_drive($conf, sub {
2459         my ($ds, $drive) = @_;
2460
2461         my $volid = $drive->{file};
2462         return if !$volid;
2463
2464         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2465         if ($storeid) {
2466             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2467             if ($scfg->{disable}) {
2468                 $nodehash = {};
2469             } elsif (my $avail = $scfg->{nodes}) {
2470                 foreach my $node (keys %$nodehash) {
2471                     delete $nodehash->{$node} if !$avail->{$node};
2472                 }
2473             } elsif (!$scfg->{shared}) {
2474                 foreach my $node (keys %$nodehash) {
2475                     delete $nodehash->{$node} if $node ne $nodename
2476                 }
2477             }
2478         }
2479     });
2480
2481     return $nodehash
2482 }
2483
2484 sub check_cmdline {
2485     my ($pidfile, $pid) = @_;
2486
2487     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2488     if (defined($fh)) {
2489         my $line = <$fh>;
2490         $fh->close;
2491         return undef if !$line;
2492         my @param = split(/\0/, $line);
2493
2494         my $cmd = $param[0];
2495         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2496
2497         for (my $i = 0; $i < scalar (@param); $i++) {
2498             my $p = $param[$i];
2499             next if !$p;
2500             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2501                 my $p = $param[$i+1];
2502                 return 1 if $p && ($p eq $pidfile);
2503                 return undef;
2504             }
2505         }
2506     }
2507     return undef;
2508 }
2509
2510 sub check_running {
2511     my ($vmid, $nocheck, $node) = @_;
2512
2513     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2514
2515     die "unable to find configuration file for VM $vmid - no such machine\n"
2516         if !$nocheck && ! -f $filename;
2517
2518     my $pidfile = pidfile_name($vmid);
2519
2520     if (my $fd = IO::File->new("<$pidfile")) {
2521         my $st = stat($fd);
2522         my $line = <$fd>;
2523         close($fd);
2524
2525         my $mtime = $st->mtime;
2526         if ($mtime > time()) {
2527             warn "file '$filename' modified in future\n";
2528         }
2529
2530         if ($line =~ m/^(\d+)$/) {
2531             my $pid = $1;
2532             if (check_cmdline($pidfile, $pid)) {
2533                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2534                     return $pid;
2535                 }
2536             }
2537         }
2538     }
2539
2540     return undef;
2541 }
2542
2543 sub vzlist {
2544
2545     my $vzlist = config_list();
2546
2547     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2548
2549     while (defined(my $de = $fd->read)) {
2550         next if $de !~ m/^(\d+)\.pid$/;
2551         my $vmid = $1;
2552         next if !defined($vzlist->{$vmid});
2553         if (my $pid = check_running($vmid)) {
2554             $vzlist->{$vmid}->{pid} = $pid;
2555         }
2556     }
2557
2558     return $vzlist;
2559 }
2560
2561 sub disksize {
2562     my ($storecfg, $conf) = @_;
2563
2564     my $bootdisk = $conf->{bootdisk};
2565     return undef if !$bootdisk;
2566     return undef if !is_valid_drivename($bootdisk);
2567
2568     return undef if !$conf->{$bootdisk};
2569
2570     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2571     return undef if !defined($drive);
2572
2573     return undef if drive_is_cdrom($drive);
2574
2575     my $volid = $drive->{file};
2576     return undef if !$volid;
2577
2578     return $drive->{size};
2579 }
2580
2581 my $last_proc_pid_stat;
2582
2583 # get VM status information
2584 # This must be fast and should not block ($full == false)
2585 # We only query KVM using QMP if $full == true (this can be slow)
2586 sub vmstatus {
2587     my ($opt_vmid, $full) = @_;
2588
2589     my $res = {};
2590
2591     my $storecfg = PVE::Storage::config();
2592
2593     my $list = vzlist();
2594     my $defaults = load_defaults();
2595
2596     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2597
2598     my $cpucount = $cpuinfo->{cpus} || 1;
2599
2600     foreach my $vmid (keys %$list) {
2601         next if $opt_vmid && ($vmid ne $opt_vmid);
2602
2603         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2604         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2605
2606         my $d = {};
2607         $d->{pid} = $list->{$vmid}->{pid};
2608
2609         # fixme: better status?
2610         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2611
2612         my $size = disksize($storecfg, $conf);
2613         if (defined($size)) {
2614             $d->{disk} = 0; # no info available
2615             $d->{maxdisk} = $size;
2616         } else {
2617             $d->{disk} = 0;
2618             $d->{maxdisk} = 0;
2619         }
2620
2621         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2622             * ($conf->{cores} || $defaults->{cores});
2623         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2624         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2625
2626         $d->{name} = $conf->{name} || "VM $vmid";
2627         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2628             : $defaults->{memory}*(1024*1024);
2629
2630         if ($conf->{balloon}) {
2631             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2632             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2633                 : $defaults->{shares};
2634         }
2635
2636         $d->{uptime} = 0;
2637         $d->{cpu} = 0;
2638         $d->{mem} = 0;
2639
2640         $d->{netout} = 0;
2641         $d->{netin} = 0;
2642
2643         $d->{diskread} = 0;
2644         $d->{diskwrite} = 0;
2645
2646         $d->{template} = PVE::QemuConfig->is_template($conf);
2647
2648         $res->{$vmid} = $d;
2649     }
2650
2651     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2652     foreach my $dev (keys %$netdev) {
2653         next if $dev !~ m/^tap([1-9]\d*)i/;
2654         my $vmid = $1;
2655         my $d = $res->{$vmid};
2656         next if !$d;
2657
2658         $d->{netout} += $netdev->{$dev}->{receive};
2659         $d->{netin} += $netdev->{$dev}->{transmit};
2660
2661         if ($full) {
2662             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2663             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2664         }
2665
2666     }
2667
2668     my $ctime = gettimeofday;
2669
2670     foreach my $vmid (keys %$list) {
2671
2672         my $d = $res->{$vmid};
2673         my $pid = $d->{pid};
2674         next if !$pid;
2675
2676         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2677         next if !$pstat; # not running
2678
2679         my $used = $pstat->{utime} + $pstat->{stime};
2680
2681         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2682
2683         if ($pstat->{vsize}) {
2684             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2685         }
2686
2687         my $old = $last_proc_pid_stat->{$pid};
2688         if (!$old) {
2689             $last_proc_pid_stat->{$pid} = {
2690                 time => $ctime,
2691                 used => $used,
2692                 cpu => 0,
2693             };
2694             next;
2695         }
2696
2697         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2698
2699         if ($dtime > 1000) {
2700             my $dutime = $used -  $old->{used};
2701
2702             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2703             $last_proc_pid_stat->{$pid} = {
2704                 time => $ctime,
2705                 used => $used,
2706                 cpu => $d->{cpu},
2707             };
2708         } else {
2709             $d->{cpu} = $old->{cpu};
2710         }
2711     }
2712
2713     return $res if !$full;
2714
2715     my $qmpclient = PVE::QMPClient->new();
2716
2717     my $ballooncb = sub {
2718         my ($vmid, $resp) = @_;
2719
2720         my $info = $resp->{'return'};
2721         return if !$info->{max_mem};
2722
2723         my $d = $res->{$vmid};
2724
2725         # use memory assigned to VM
2726         $d->{maxmem} = $info->{max_mem};
2727         $d->{balloon} = $info->{actual};
2728
2729         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2730             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2731             $d->{freemem} = $info->{free_mem};
2732         }
2733
2734         $d->{ballooninfo} = $info;
2735     };
2736
2737     my $blockstatscb = sub {
2738         my ($vmid, $resp) = @_;
2739         my $data = $resp->{'return'} || [];
2740         my $totalrdbytes = 0;
2741         my $totalwrbytes = 0;
2742
2743         for my $blockstat (@$data) {
2744             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2745             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2746
2747             $blockstat->{device} =~ s/drive-//;
2748             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2749         }
2750         $res->{$vmid}->{diskread} = $totalrdbytes;
2751         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2752     };
2753
2754     my $statuscb = sub {
2755         my ($vmid, $resp) = @_;
2756
2757         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2758         # this fails if ballon driver is not loaded, so this must be
2759         # the last commnand (following command are aborted if this fails).
2760         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2761
2762         my $status = 'unknown';
2763         if (!defined($status = $resp->{'return'}->{status})) {
2764             warn "unable to get VM status\n";
2765             return;
2766         }
2767
2768         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2769     };
2770
2771     foreach my $vmid (keys %$list) {
2772         next if $opt_vmid && ($vmid ne $opt_vmid);
2773         next if !$res->{$vmid}->{pid}; # not running
2774         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2775     }
2776
2777     $qmpclient->queue_execute(undef, 2);
2778
2779     foreach my $vmid (keys %$list) {
2780         next if $opt_vmid && ($vmid ne $opt_vmid);
2781         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2782     }
2783
2784     return $res;
2785 }
2786
2787 sub foreach_drive {
2788     my ($conf, $func, @param) = @_;
2789
2790     foreach my $ds (valid_drive_names()) {
2791         next if !defined($conf->{$ds});
2792
2793         my $drive = parse_drive($ds, $conf->{$ds});
2794         next if !$drive;
2795
2796         &$func($ds, $drive, @param);
2797     }
2798 }
2799
2800 sub foreach_volid {
2801     my ($conf, $func, @param) = @_;
2802
2803     my $volhash = {};
2804
2805     my $test_volid = sub {
2806         my ($volid, $is_cdrom, $replicate, $snapname) = @_;
2807
2808         return if !$volid;
2809
2810         $volhash->{$volid}->{cdrom} //= 1;
2811         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2812
2813         $volhash->{$volid}->{replicate} //= 0;
2814         $volhash->{$volid}->{replicate} = 1 if $replicate;
2815
2816         $volhash->{$volid}->{referenced_in_config} //= 0;
2817         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
2818
2819         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
2820             if defined($snapname);
2821     };
2822
2823     foreach_drive($conf, sub {
2824         my ($ds, $drive) = @_;
2825         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, undef);
2826     });
2827
2828     foreach my $snapname (keys %{$conf->{snapshots}}) {
2829         my $snap = $conf->{snapshots}->{$snapname};
2830         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
2831         foreach_drive($snap, sub {
2832             my ($ds, $drive) = @_;
2833             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $snapname);
2834         });
2835     }
2836
2837     foreach my $volid (keys %$volhash) {
2838         &$func($volid, $volhash->{$volid}, @param);
2839     }
2840 }
2841
2842 sub vga_conf_has_spice {
2843     my ($vga) = @_;
2844
2845     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2846
2847     return $1 || 1;
2848 }
2849
2850 sub config_to_command {
2851     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2852
2853     my $cmd = [];
2854     my $globalFlags = [];
2855     my $machineFlags = [];
2856     my $rtcFlags = [];
2857     my $cpuFlags = [];
2858     my $devices = [];
2859     my $pciaddr = '';
2860     my $bridges = {};
2861     my $kvmver = kvm_user_version();
2862     my $vernum = 0; # unknown
2863     my $ostype = $conf->{ostype};
2864     my $winversion = windows_version($ostype);
2865     my $kvm = $conf->{kvm} // 1;
2866
2867     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
2868
2869     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2870         $vernum = $1*1000000+$2*1000;
2871     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2872         $vernum = $1*1000000+$2*1000+$3;
2873     }
2874
2875     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2876
2877     my $have_ovz = -f '/proc/vz/vestat';
2878
2879     my $q35 = machine_type_is_q35($conf);
2880     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2881     my $machine_type = $forcemachine || $conf->{machine};
2882     my $use_old_bios_files = undef;
2883     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2884
2885     my $cpuunits = defined($conf->{cpuunits}) ?
2886             $conf->{cpuunits} : $defaults->{cpuunits};
2887
2888     push @$cmd, '/usr/bin/kvm';
2889
2890     push @$cmd, '-id', $vmid;
2891
2892     my $use_virtio = 0;
2893
2894     my $qmpsocket = qmp_socket($vmid);
2895     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2896     push @$cmd, '-mon', "chardev=qmp,mode=control";
2897
2898
2899     push @$cmd, '-pidfile' , pidfile_name($vmid);
2900
2901     push @$cmd, '-daemonize';
2902
2903     if ($conf->{smbios1}) {
2904         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2905     }
2906
2907     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2908         die "uefi base image not found\n" if ! -f $OVMF_CODE;
2909
2910         my $path;
2911         my $format;
2912         if (my $efidisk = $conf->{efidisk0}) {
2913             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
2914             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2915             $format = $d->{format};
2916             if ($storeid) {
2917                 $path = PVE::Storage::path($storecfg, $d->{file});
2918                 if (!defined($format)) {
2919                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2920                     $format = qemu_img_format($scfg, $volname);
2921                 }
2922             } else {
2923                 $path = $d->{file};
2924                 die "efidisk format must be specified\n"
2925                     if !defined($format);
2926             }
2927         } else {
2928             warn "no efidisk configured! Using temporary efivars disk.\n";
2929             $path = "/tmp/$vmid-ovmf.fd";
2930             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
2931             $format = 'raw';
2932         }
2933
2934         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
2935         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
2936     }
2937
2938
2939     # add usb controllers
2940     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2941     push @$devices, @usbcontrollers if @usbcontrollers;
2942     my $vga = $conf->{vga};
2943
2944     my $qxlnum = vga_conf_has_spice($vga);
2945     $vga = 'qxl' if $qxlnum;
2946
2947     if (!$vga) {
2948         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
2949             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
2950         } else {
2951             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
2952         }
2953     }
2954
2955     # enable absolute mouse coordinates (needed by vnc)
2956     my $tablet;
2957     if (defined($conf->{tablet})) {
2958         $tablet = $conf->{tablet};
2959     } else {
2960         $tablet = $defaults->{tablet};
2961         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2962         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2963     }
2964
2965     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2966
2967     my $kvm_off = 0;
2968     my $gpu_passthrough;
2969
2970     # host pci devices
2971     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2972         my $d = parse_hostpci($conf->{"hostpci$i"});
2973         next if !$d;
2974
2975         my $pcie = $d->{pcie};
2976         if($pcie){
2977             die "q35 machine model is not enabled" if !$q35;
2978             $pciaddr = print_pcie_addr("hostpci$i");
2979         }else{
2980             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2981         }
2982
2983         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2984         my $romfile = $d->{romfile};
2985
2986         my $xvga = '';
2987         if ($d->{'x-vga'}) {
2988             $xvga = ',x-vga=on';
2989             $kvm_off = 1;
2990             $vga = 'none';
2991             $gpu_passthrough = 1;
2992
2993             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2994                 $xvga = "";
2995             }
2996         }
2997         my $pcidevices = $d->{pciid};
2998         my $multifunction = 1 if @$pcidevices > 1;
2999
3000         my $j=0;
3001         foreach my $pcidevice (@$pcidevices) {
3002
3003             my $id = "hostpci$i";
3004             $id .= ".$j" if $multifunction;
3005             my $addr = $pciaddr;
3006             $addr .= ".$j" if $multifunction;
3007             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3008
3009             if($j == 0){
3010                 $devicestr .= "$rombar$xvga";
3011                 $devicestr .= ",multifunction=on" if $multifunction;
3012                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3013             }
3014
3015             push @$devices, '-device', $devicestr;
3016             $j++;
3017         }
3018     }
3019
3020     # usb devices
3021     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3022     push @$devices, @usbdevices if @usbdevices;
3023     # serial devices
3024     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3025         if (my $path = $conf->{"serial$i"}) {
3026             if ($path eq 'socket') {
3027                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3028                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3029                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3030             } else {
3031                 die "no such serial device\n" if ! -c $path;
3032                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3033                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3034             }
3035         }
3036     }
3037
3038     # parallel devices
3039     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3040         if (my $path = $conf->{"parallel$i"}) {
3041             die "no such parallel device\n" if ! -c $path;
3042             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3043             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3044             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3045         }
3046     }
3047
3048     my $vmname = $conf->{name} || "vm$vmid";
3049
3050     push @$cmd, '-name', $vmname;
3051
3052     my $sockets = 1;
3053     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3054     $sockets = $conf->{sockets} if  $conf->{sockets};
3055
3056     my $cores = $conf->{cores} || 1;
3057
3058     my $maxcpus = $sockets * $cores;
3059
3060     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3061
3062     my $allowed_vcpus = $cpuinfo->{cpus};
3063
3064     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3065         if ($allowed_vcpus < $maxcpus);
3066
3067     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3068
3069         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3070         for (my $i = 2; $i <= $vcpus; $i++)  {
3071             my $cpustr = print_cpu_device($conf,$i);
3072             push @$cmd, '-device', $cpustr;
3073         }
3074
3075     } else {
3076
3077         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3078     }
3079     push @$cmd, '-nodefaults';
3080
3081     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3082
3083     my $bootindex_hash = {};
3084     my $i = 1;
3085     foreach my $o (split(//, $bootorder)) {
3086         $bootindex_hash->{$o} = $i*100;
3087         $i++;
3088     }
3089
3090     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3091
3092     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3093
3094     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3095
3096     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3097
3098     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3099         my $socket = vnc_socket($vmid);
3100         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3101     } else {
3102         push @$cmd, '-nographic';
3103     }
3104
3105     # time drift fix
3106     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3107
3108     my $useLocaltime = $conf->{localtime};
3109
3110     if ($winversion >= 5) { # windows
3111         $useLocaltime = 1 if !defined($conf->{localtime});
3112
3113         # use time drift fix when acpi is enabled
3114         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3115             $tdf = 1 if !defined($conf->{tdf});
3116         }
3117     }
3118
3119     if ($winversion >= 6) {
3120         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3121         push @$cmd, '-no-hpet';
3122     }
3123
3124     push @$rtcFlags, 'driftfix=slew' if $tdf;
3125
3126     if (!$kvm) {
3127         push @$machineFlags, 'accel=tcg';
3128     }
3129
3130     if ($machine_type) {
3131         push @$machineFlags, "type=${machine_type}";
3132     }
3133
3134     if ($conf->{startdate}) {
3135         push @$rtcFlags, "base=$conf->{startdate}";
3136     } elsif ($useLocaltime) {
3137         push @$rtcFlags, 'base=localtime';
3138     }
3139
3140     my $cpu = $kvm ? "kvm64" : "qemu64";
3141     if (my $cputype = $conf->{cpu}) {
3142         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3143             or die "Cannot parse cpu description: $cputype\n";
3144         $cpu = $cpuconf->{cputype};
3145         $kvm_off = 1 if $cpuconf->{hidden};
3146     }
3147
3148     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3149
3150     push @$cpuFlags , '-x2apic'
3151         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3152
3153     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3154
3155     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3156
3157     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3158
3159         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3160         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3161     }
3162
3163     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3164
3165     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3166
3167     push @$cpuFlags, 'kvm=off' if $kvm_off;
3168
3169     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3170         die "internal error"; # should not happen
3171
3172     push @$cpuFlags, "vendor=${cpu_vendor}"
3173         if $cpu_vendor ne 'default';
3174
3175     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3176
3177     push @$cmd, '-cpu', $cpu;
3178
3179     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3180
3181     push @$cmd, '-S' if $conf->{freeze};
3182
3183     # set keyboard layout
3184     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3185     push @$cmd, '-k', $kb if $kb;
3186
3187     # enable sound
3188     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3189     #push @$cmd, '-soundhw', 'es1370';
3190     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3191
3192     if($conf->{agent}) {
3193         my $qgasocket = qmp_socket($vmid, 1);
3194         my $pciaddr = print_pci_addr("qga0", $bridges);
3195         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3196         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3197         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3198     }
3199
3200     my $spice_port;
3201
3202     if ($qxlnum) {
3203         if ($qxlnum > 1) {
3204             if ($winversion){
3205                 for(my $i = 1; $i < $qxlnum; $i++){
3206                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3207                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3208                 }
3209             } else {
3210                 # assume other OS works like Linux
3211                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3212                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3213             }
3214         }
3215
3216         my $pciaddr = print_pci_addr("spice", $bridges);
3217
3218         my $nodename = PVE::INotify::nodename();
3219         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3220         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3221         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3222         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3223         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3224
3225         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3226
3227         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3228         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3229         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3230     }
3231
3232     # enable balloon by default, unless explicitly disabled
3233     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3234         $pciaddr = print_pci_addr("balloon0", $bridges);
3235         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3236     }
3237
3238     if ($conf->{watchdog}) {
3239         my $wdopts = parse_watchdog($conf->{watchdog});
3240         $pciaddr = print_pci_addr("watchdog", $bridges);
3241         my $watchdog = $wdopts->{model} || 'i6300esb';
3242         push @$devices, '-device', "$watchdog$pciaddr";
3243         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3244     }
3245
3246     my $vollist = [];
3247     my $scsicontroller = {};
3248     my $ahcicontroller = {};
3249     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3250
3251     # Add iscsi initiator name if available
3252     if (my $initiator = get_initiator_name()) {
3253         push @$devices, '-iscsi', "initiator-name=$initiator";
3254     }
3255
3256     foreach_drive($conf, sub {
3257         my ($ds, $drive) = @_;
3258
3259         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3260             push @$vollist, $drive->{file};
3261         }
3262
3263         # ignore efidisk here, already added in bios/fw handling code above
3264         return if $drive->{interface} eq 'efidisk';
3265
3266         $use_virtio = 1 if $ds =~ m/^virtio/;
3267
3268         if (drive_is_cdrom ($drive)) {
3269             if ($bootindex_hash->{d}) {
3270                 $drive->{bootindex} = $bootindex_hash->{d};
3271                 $bootindex_hash->{d} += 1;
3272             }
3273         } else {
3274             if ($bootindex_hash->{c}) {
3275                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3276                 $bootindex_hash->{c} += 1;
3277             }
3278         }
3279
3280         if($drive->{interface} eq 'virtio'){
3281            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3282         }
3283
3284         if ($drive->{interface} eq 'scsi') {
3285
3286             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3287
3288             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3289             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3290
3291             my $iothread = '';
3292             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3293                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3294                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3295             } elsif ($drive->{iothread}) {
3296                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3297             }
3298
3299             my $queues = '';
3300             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3301                 $queues = ",num_queues=$drive->{queues}";
3302             }
3303
3304             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3305             $scsicontroller->{$controller}=1;
3306         }
3307
3308         if ($drive->{interface} eq 'sata') {
3309            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3310            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3311            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3312            $ahcicontroller->{$controller}=1;
3313         }
3314
3315         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3316         push @$devices, '-drive',$drive_cmd;
3317         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3318     });
3319
3320     for (my $i = 0; $i < $MAX_NETS; $i++) {
3321          next if !$conf->{"net$i"};
3322          my $d = parse_net($conf->{"net$i"});
3323          next if !$d;
3324
3325          $use_virtio = 1 if $d->{model} eq 'virtio';
3326
3327          if ($bootindex_hash->{n}) {
3328             $d->{bootindex} = $bootindex_hash->{n};
3329             $bootindex_hash->{n} += 1;
3330          }
3331
3332          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3333          push @$devices, '-netdev', $netdevfull;
3334
3335          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3336          push @$devices, '-device', $netdevicefull;
3337     }
3338
3339     if (!$q35) {
3340         # add pci bridges
3341         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3342            $bridges->{1} = 1;
3343            $bridges->{2} = 1;
3344         }
3345
3346         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3347
3348         while (my ($k, $v) = each %$bridges) {
3349             $pciaddr = print_pci_addr("pci.$k");
3350             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3351         }
3352     }
3353
3354     # add custom args
3355     if ($conf->{args}) {
3356         my $aa = PVE::Tools::split_args($conf->{args});
3357         push @$cmd, @$aa;
3358     }
3359
3360     push @$cmd, @$devices;
3361     push @$cmd, '-rtc', join(',', @$rtcFlags)
3362         if scalar(@$rtcFlags);
3363     push @$cmd, '-machine', join(',', @$machineFlags)
3364         if scalar(@$machineFlags);
3365     push @$cmd, '-global', join(',', @$globalFlags)
3366         if scalar(@$globalFlags);
3367
3368     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3369 }
3370
3371 sub vnc_socket {
3372     my ($vmid) = @_;
3373     return "${var_run_tmpdir}/$vmid.vnc";
3374 }
3375
3376 sub spice_port {
3377     my ($vmid) = @_;
3378
3379     my $res = vm_mon_cmd($vmid, 'query-spice');
3380
3381     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3382 }
3383
3384 sub qmp_socket {
3385     my ($vmid, $qga) = @_;
3386     my $sockettype = $qga ? 'qga' : 'qmp';
3387     return "${var_run_tmpdir}/$vmid.$sockettype";
3388 }
3389
3390 sub pidfile_name {
3391     my ($vmid) = @_;
3392     return "${var_run_tmpdir}/$vmid.pid";
3393 }
3394
3395 sub vm_devices_list {
3396     my ($vmid) = @_;
3397
3398     my $res = vm_mon_cmd($vmid, 'query-pci');
3399     my $devices = {};
3400     foreach my $pcibus (@$res) {
3401         foreach my $device (@{$pcibus->{devices}}) {
3402             next if !$device->{'qdev_id'};
3403             if ($device->{'pci_bridge'}) {
3404                 $devices->{$device->{'qdev_id'}} = 1;
3405                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3406                     next if !$bridge_device->{'qdev_id'};
3407                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3408                     $devices->{$device->{'qdev_id'}}++;
3409                 }
3410             } else {
3411                 $devices->{$device->{'qdev_id'}} = 1;
3412             }
3413         }
3414     }
3415
3416     my $resblock = vm_mon_cmd($vmid, 'query-block');
3417     foreach my $block (@$resblock) {
3418         if($block->{device} =~ m/^drive-(\S+)/){
3419                 $devices->{$1} = 1;
3420         }
3421     }
3422
3423     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3424     foreach my $mice (@$resmice) {
3425         if ($mice->{name} eq 'QEMU HID Tablet') {
3426             $devices->{tablet} = 1;
3427             last;
3428         }
3429     }
3430
3431     # for usb devices there is no query-usb
3432     # but we can iterate over the entries in
3433     # qom-list path=/machine/peripheral
3434     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3435     foreach my $per (@$resperipheral) {
3436         if ($per->{name} =~ m/^usb\d+$/) {
3437             $devices->{$per->{name}} = 1;
3438         }
3439     }
3440
3441     return $devices;
3442 }
3443
3444 sub vm_deviceplug {
3445     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3446
3447     my $q35 = machine_type_is_q35($conf);
3448
3449     my $devices_list = vm_devices_list($vmid);
3450     return 1 if defined($devices_list->{$deviceid});
3451
3452     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3453
3454     if ($deviceid eq 'tablet') {
3455
3456         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3457
3458     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3459
3460         die "usb hotplug currently not reliable\n";
3461         # since we can't reliably hot unplug all added usb devices
3462         # and usb passthrough disables live migration
3463         # we disable usb hotplugging for now
3464         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3465
3466     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3467
3468         qemu_iothread_add($vmid, $deviceid, $device);
3469
3470         qemu_driveadd($storecfg, $vmid, $device);
3471         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3472
3473         qemu_deviceadd($vmid, $devicefull);
3474         eval { qemu_deviceaddverify($vmid, $deviceid); };
3475         if (my $err = $@) {
3476             eval { qemu_drivedel($vmid, $deviceid); };
3477             warn $@ if $@;
3478             die $err;
3479         }
3480
3481     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3482
3483
3484         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3485         my $pciaddr = print_pci_addr($deviceid);
3486         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3487
3488         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3489
3490         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3491             qemu_iothread_add($vmid, $deviceid, $device);
3492             $devicefull .= ",iothread=iothread-$deviceid";
3493         }
3494
3495         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3496             $devicefull .= ",num_queues=$device->{queues}";
3497         }
3498
3499         qemu_deviceadd($vmid, $devicefull);
3500         qemu_deviceaddverify($vmid, $deviceid);
3501
3502     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3503
3504         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3505         qemu_driveadd($storecfg, $vmid, $device);
3506
3507         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3508         eval { qemu_deviceadd($vmid, $devicefull); };
3509         if (my $err = $@) {
3510             eval { qemu_drivedel($vmid, $deviceid); };
3511             warn $@ if $@;
3512             die $err;
3513         }
3514
3515     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3516
3517         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3518
3519         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3520         my $use_old_bios_files = undef;
3521         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3522
3523         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3524         qemu_deviceadd($vmid, $netdevicefull);
3525         eval { qemu_deviceaddverify($vmid, $deviceid); };
3526         if (my $err = $@) {
3527             eval { qemu_netdevdel($vmid, $deviceid); };
3528             warn $@ if $@;
3529             die $err;
3530         }
3531
3532     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3533
3534         my $bridgeid = $2;
3535         my $pciaddr = print_pci_addr($deviceid);
3536         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3537
3538         qemu_deviceadd($vmid, $devicefull);
3539         qemu_deviceaddverify($vmid, $deviceid);
3540
3541     } else {
3542         die "can't hotplug device '$deviceid'\n";
3543     }
3544
3545     return 1;
3546 }
3547
3548 # fixme: this should raise exceptions on error!
3549 sub vm_deviceunplug {
3550     my ($vmid, $conf, $deviceid) = @_;
3551
3552     my $devices_list = vm_devices_list($vmid);
3553     return 1 if !defined($devices_list->{$deviceid});
3554
3555     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3556
3557     if ($deviceid eq 'tablet') {
3558
3559         qemu_devicedel($vmid, $deviceid);
3560
3561     } elsif ($deviceid =~ m/^usb\d+$/) {
3562
3563         die "usb hotplug currently not reliable\n";
3564         # when unplugging usb devices this way,
3565         # there may be remaining usb controllers/hubs
3566         # so we disable it for now
3567         qemu_devicedel($vmid, $deviceid);
3568         qemu_devicedelverify($vmid, $deviceid);
3569
3570     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3571
3572         qemu_devicedel($vmid, $deviceid);
3573         qemu_devicedelverify($vmid, $deviceid);
3574         qemu_drivedel($vmid, $deviceid);
3575         qemu_iothread_del($conf, $vmid, $deviceid);
3576
3577     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3578
3579         qemu_devicedel($vmid, $deviceid);
3580         qemu_devicedelverify($vmid, $deviceid);
3581         qemu_iothread_del($conf, $vmid, $deviceid);
3582
3583     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3584
3585         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3586         my $device = parse_drive($deviceid, $conf->{$deviceid});
3587         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3588
3589         qemu_devicedel($vmid, $deviceid);
3590         qemu_drivedel($vmid, $deviceid);
3591         qemu_deletescsihw($conf, $vmid, $deviceid);
3592
3593     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3594
3595         qemu_devicedel($vmid, $deviceid);
3596         qemu_devicedelverify($vmid, $deviceid);
3597         qemu_netdevdel($vmid, $deviceid);
3598
3599     } else {
3600         die "can't unplug device '$deviceid'\n";
3601     }
3602
3603     return 1;
3604 }
3605
3606 sub qemu_deviceadd {
3607     my ($vmid, $devicefull) = @_;
3608
3609     $devicefull = "driver=".$devicefull;
3610     my %options =  split(/[=,]/, $devicefull);
3611
3612     vm_mon_cmd($vmid, "device_add" , %options);
3613 }
3614
3615 sub qemu_devicedel {
3616     my ($vmid, $deviceid) = @_;
3617
3618     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3619 }
3620
3621 sub qemu_iothread_add {
3622     my($vmid, $deviceid, $device) = @_;
3623
3624     if ($device->{iothread}) {
3625         my $iothreads = vm_iothreads_list($vmid);
3626         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3627     }
3628 }
3629
3630 sub qemu_iothread_del {
3631     my($conf, $vmid, $deviceid) = @_;
3632
3633     my $device = parse_drive($deviceid, $conf->{$deviceid});
3634     if ($device->{iothread}) {
3635         my $iothreads = vm_iothreads_list($vmid);
3636         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3637     }
3638 }
3639
3640 sub qemu_objectadd {
3641     my($vmid, $objectid, $qomtype) = @_;
3642
3643     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3644
3645     return 1;
3646 }
3647
3648 sub qemu_objectdel {
3649     my($vmid, $objectid) = @_;
3650
3651     vm_mon_cmd($vmid, "object-del", id => $objectid);
3652
3653     return 1;
3654 }
3655
3656 sub qemu_driveadd {
3657     my ($storecfg, $vmid, $device) = @_;
3658
3659     my $drive = print_drive_full($storecfg, $vmid, $device);
3660     $drive =~ s/\\/\\\\/g;
3661     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3662
3663     # If the command succeeds qemu prints: "OK"
3664     return 1 if $ret =~ m/OK/s;
3665
3666     die "adding drive failed: $ret\n";
3667 }
3668
3669 sub qemu_drivedel {
3670     my($vmid, $deviceid) = @_;
3671
3672     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3673     $ret =~ s/^\s+//;
3674
3675     return 1 if $ret eq "";
3676
3677     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3678     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3679
3680     die "deleting drive $deviceid failed : $ret\n";
3681 }
3682
3683 sub qemu_deviceaddverify {
3684     my ($vmid, $deviceid) = @_;
3685
3686     for (my $i = 0; $i <= 5; $i++) {
3687          my $devices_list = vm_devices_list($vmid);
3688          return 1 if defined($devices_list->{$deviceid});
3689          sleep 1;
3690     }
3691
3692     die "error on hotplug device '$deviceid'\n";
3693 }
3694
3695
3696 sub qemu_devicedelverify {
3697     my ($vmid, $deviceid) = @_;
3698
3699     # need to verify that the device is correctly removed as device_del
3700     # is async and empty return is not reliable
3701
3702     for (my $i = 0; $i <= 5; $i++) {
3703          my $devices_list = vm_devices_list($vmid);
3704          return 1 if !defined($devices_list->{$deviceid});
3705          sleep 1;
3706     }
3707
3708     die "error on hot-unplugging device '$deviceid'\n";
3709 }
3710
3711 sub qemu_findorcreatescsihw {
3712     my ($storecfg, $conf, $vmid, $device) = @_;
3713
3714     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3715
3716     my $scsihwid="$controller_prefix$controller";
3717     my $devices_list = vm_devices_list($vmid);
3718
3719     if(!defined($devices_list->{$scsihwid})) {
3720         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3721     }
3722
3723     return 1;
3724 }
3725
3726 sub qemu_deletescsihw {
3727     my ($conf, $vmid, $opt) = @_;
3728
3729     my $device = parse_drive($opt, $conf->{$opt});
3730
3731     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3732         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3733         return 1;
3734     }
3735
3736     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3737
3738     my $devices_list = vm_devices_list($vmid);
3739     foreach my $opt (keys %{$devices_list}) {
3740         if (PVE::QemuServer::is_valid_drivename($opt)) {
3741             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3742             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3743                 return 1;
3744             }
3745         }
3746     }
3747
3748     my $scsihwid="scsihw$controller";
3749
3750     vm_deviceunplug($vmid, $conf, $scsihwid);
3751
3752     return 1;
3753 }
3754
3755 sub qemu_add_pci_bridge {
3756     my ($storecfg, $conf, $vmid, $device) = @_;
3757
3758     my $bridges = {};
3759
3760     my $bridgeid;
3761
3762     print_pci_addr($device, $bridges);
3763
3764     while (my ($k, $v) = each %$bridges) {
3765         $bridgeid = $k;
3766     }
3767     return 1 if !defined($bridgeid) || $bridgeid < 1;
3768
3769     my $bridge = "pci.$bridgeid";
3770     my $devices_list = vm_devices_list($vmid);
3771
3772     if (!defined($devices_list->{$bridge})) {
3773         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3774     }
3775
3776     return 1;
3777 }
3778
3779 sub qemu_set_link_status {
3780     my ($vmid, $device, $up) = @_;
3781
3782     vm_mon_cmd($vmid, "set_link", name => $device,
3783                up => $up ? JSON::true : JSON::false);
3784 }
3785
3786 sub qemu_netdevadd {
3787     my ($vmid, $conf, $device, $deviceid) = @_;
3788
3789     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3790     my %options =  split(/[=,]/, $netdev);
3791
3792     vm_mon_cmd($vmid, "netdev_add",  %options);
3793     return 1;
3794 }
3795
3796 sub qemu_netdevdel {
3797     my ($vmid, $deviceid) = @_;
3798
3799     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3800 }
3801
3802 sub qemu_usb_hotplug {
3803     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3804
3805     return if !$device;
3806
3807     # remove the old one first
3808     vm_deviceunplug($vmid, $conf, $deviceid);
3809
3810     # check if xhci controller is necessary and available
3811     if ($device->{usb3}) {
3812
3813         my $devicelist = vm_devices_list($vmid);
3814
3815         if (!$devicelist->{xhci}) {
3816             my $pciaddr = print_pci_addr("xhci");
3817             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3818         }
3819     }
3820     my $d = parse_usb_device($device->{host});
3821     $d->{usb3} = $device->{usb3};
3822
3823     # add the new one
3824     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3825 }
3826
3827 sub qemu_cpu_hotplug {
3828     my ($vmid, $conf, $vcpus) = @_;
3829
3830     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3831
3832     my $sockets = 1;
3833     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3834     $sockets = $conf->{sockets} if  $conf->{sockets};
3835     my $cores = $conf->{cores} || 1;
3836     my $maxcpus = $sockets * $cores;
3837
3838     $vcpus = $maxcpus if !$vcpus;
3839
3840     die "you can't add more vcpus than maxcpus\n"
3841         if $vcpus > $maxcpus;
3842
3843     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3844
3845     if ($vcpus < $currentvcpus) {
3846
3847         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3848
3849             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3850                 qemu_devicedel($vmid, "cpu$i");
3851                 my $retry = 0;
3852                 my $currentrunningvcpus = undef;
3853                 while (1) {
3854                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3855                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3856                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3857                     $retry++;
3858                     sleep 1;
3859                 }
3860                 #update conf after each succesfull cpu unplug
3861                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3862                 PVE::QemuConfig->write_config($vmid, $conf);
3863             }
3864         } else {
3865             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3866         }
3867
3868         return;
3869     }
3870
3871     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3872     die "vcpus in running vm does not match its configuration\n"
3873         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3874
3875     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3876
3877         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3878             my $cpustr = print_cpu_device($conf, $i);
3879             qemu_deviceadd($vmid, $cpustr);
3880
3881             my $retry = 0;
3882             my $currentrunningvcpus = undef;
3883             while (1) {
3884                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3885                 last if scalar(@{$currentrunningvcpus}) == $i;
3886                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3887                 sleep 1;
3888                 $retry++;
3889             }
3890             #update conf after each succesfull cpu hotplug
3891             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3892             PVE::QemuConfig->write_config($vmid, $conf);
3893         }
3894     } else {
3895
3896         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3897             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3898         }
3899     }
3900 }
3901
3902 sub qemu_block_set_io_throttle {
3903     my ($vmid, $deviceid,
3904         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3905         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3906         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3907         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3908
3909     return if !check_running($vmid) ;
3910
3911     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3912         bps => int($bps),
3913         bps_rd => int($bps_rd),
3914         bps_wr => int($bps_wr),
3915         iops => int($iops),
3916         iops_rd => int($iops_rd),
3917         iops_wr => int($iops_wr),
3918         bps_max => int($bps_max),
3919         bps_rd_max => int($bps_rd_max),
3920         bps_wr_max => int($bps_wr_max),
3921         iops_max => int($iops_max),
3922         iops_rd_max => int($iops_rd_max),
3923         iops_wr_max => int($iops_wr_max),
3924         bps_max_length => int($bps_max_length),
3925         bps_rd_max_length => int($bps_rd_max_length),
3926         bps_wr_max_length => int($bps_wr_max_length),
3927         iops_max_length => int($iops_max_length),
3928         iops_rd_max_length => int($iops_rd_max_length),
3929         iops_wr_max_length => int($iops_wr_max_length),
3930     );
3931
3932 }
3933
3934 # old code, only used to shutdown old VM after update
3935 sub __read_avail {
3936     my ($fh, $timeout) = @_;
3937
3938     my $sel = new IO::Select;
3939     $sel->add($fh);
3940
3941     my $res = '';
3942     my $buf;
3943
3944     my @ready;
3945     while (scalar (@ready = $sel->can_read($timeout))) {
3946         my $count;
3947         if ($count = $fh->sysread($buf, 8192)) {
3948             if ($buf =~ /^(.*)\(qemu\) $/s) {
3949                 $res .= $1;
3950                 last;
3951             } else {
3952                 $res .= $buf;
3953             }
3954         } else {
3955             if (!defined($count)) {
3956                 die "$!\n";
3957             }
3958             last;
3959         }
3960     }
3961
3962     die "monitor read timeout\n" if !scalar(@ready);
3963
3964     return $res;
3965 }
3966
3967 # old code, only used to shutdown old VM after update
3968 sub vm_monitor_command {
3969     my ($vmid, $cmdstr, $nocheck) = @_;
3970
3971     my $res;
3972
3973     eval {
3974         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3975
3976         my $sname = "${var_run_tmpdir}/$vmid.mon";
3977
3978         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3979             die "unable to connect to VM $vmid socket - $!\n";
3980
3981         my $timeout = 3;
3982
3983         # hack: migrate sometime blocks the monitor (when migrate_downtime
3984         # is set)
3985         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3986             $timeout = 60*60; # 1 hour
3987         }
3988
3989         # read banner;
3990         my $data = __read_avail($sock, $timeout);
3991
3992         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3993             die "got unexpected qemu monitor banner\n";
3994         }
3995
3996         my $sel = new IO::Select;
3997         $sel->add($sock);
3998
3999         if (!scalar(my @ready = $sel->can_write($timeout))) {
4000             die "monitor write error - timeout";
4001         }
4002
4003         my $fullcmd = "$cmdstr\r";
4004
4005         # syslog('info', "VM $vmid monitor command: $cmdstr");
4006
4007         my $b;
4008         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
4009             die "monitor write error - $!";
4010         }
4011
4012         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4013
4014         $timeout = 20;
4015
4016         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4017             $timeout = 60*60; # 1 hour
4018         } elsif ($cmdstr =~ m/^(eject|change)/) {
4019             $timeout = 60; # note: cdrom mount command is slow
4020         }
4021         if ($res = __read_avail($sock, $timeout)) {
4022
4023             my @lines = split("\r?\n", $res);
4024
4025             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4026
4027             $res = join("\n", @lines);
4028             $res .= "\n";
4029         }
4030     };
4031
4032     my $err = $@;
4033
4034     if ($err) {
4035         syslog("err", "VM $vmid monitor command failed - $err");
4036         die $err;
4037     }
4038
4039     return $res;
4040 }
4041
4042 sub qemu_block_resize {
4043     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4044
4045     my $running = check_running($vmid);
4046
4047     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4048
4049     return if !$running;
4050
4051     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4052
4053 }
4054
4055 sub qemu_volume_snapshot {
4056     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4057
4058     my $running = check_running($vmid);
4059
4060     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4061         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4062     } else {
4063         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4064     }
4065 }
4066
4067 sub qemu_volume_snapshot_delete {
4068     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4069
4070     my $running = check_running($vmid);
4071
4072     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4073         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4074     } else {
4075         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4076     }
4077 }
4078
4079 sub set_migration_caps {
4080     my ($vmid) = @_;
4081
4082     my $cap_ref = [];
4083
4084     my $enabled_cap = {
4085         "auto-converge" => 1,
4086         "xbzrle" => 1,
4087         "x-rdma-pin-all" => 0,
4088         "zero-blocks" => 0,
4089         "compress" => 0
4090     };
4091
4092     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4093
4094     for my $supported_capability (@$supported_capabilities) {
4095         push @$cap_ref, {
4096             capability => $supported_capability->{capability},
4097             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4098         };
4099     }
4100
4101     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4102 }
4103
4104 my $fast_plug_option = {
4105     'lock' => 1,
4106     'name' => 1,
4107     'onboot' => 1,
4108     'shares' => 1,
4109     'startup' => 1,
4110     'description' => 1,
4111     'protection' => 1,
4112     'vmstatestorage' => 1,
4113 };
4114
4115 # hotplug changes in [PENDING]
4116 # $selection hash can be used to only apply specified options, for
4117 # example: { cores => 1 } (only apply changed 'cores')
4118 # $errors ref is used to return error messages
4119 sub vmconfig_hotplug_pending {
4120     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4121
4122     my $defaults = load_defaults();
4123
4124     # commit values which do not have any impact on running VM first
4125     # Note: those option cannot raise errors, we we do not care about
4126     # $selection and always apply them.
4127
4128     my $add_error = sub {
4129         my ($opt, $msg) = @_;
4130         $errors->{$opt} = "hotplug problem - $msg";
4131     };
4132
4133     my $changes = 0;
4134     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4135         if ($fast_plug_option->{$opt}) {
4136             $conf->{$opt} = $conf->{pending}->{$opt};
4137             delete $conf->{pending}->{$opt};
4138             $changes = 1;
4139         }
4140     }
4141
4142     if ($changes) {
4143         PVE::QemuConfig->write_config($vmid, $conf);
4144         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4145     }
4146
4147     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4148
4149     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4150     while (my ($opt, $force) = each %$pending_delete_hash) {
4151         next if $selection && !$selection->{$opt};
4152         eval {
4153             if ($opt eq 'hotplug') {
4154                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4155             } elsif ($opt eq 'tablet') {
4156                 die "skip\n" if !$hotplug_features->{usb};
4157                 if ($defaults->{tablet}) {
4158                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4159                 } else {
4160                     vm_deviceunplug($vmid, $conf, $opt);
4161                 }
4162             } elsif ($opt =~ m/^usb\d+/) {
4163                 die "skip\n";
4164                 # since we cannot reliably hot unplug usb devices
4165                 # we are disabling it
4166                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4167                 vm_deviceunplug($vmid, $conf, $opt);
4168             } elsif ($opt eq 'vcpus') {
4169                 die "skip\n" if !$hotplug_features->{cpu};
4170                 qemu_cpu_hotplug($vmid, $conf, undef);
4171             } elsif ($opt eq 'balloon') {
4172                 # enable balloon device is not hotpluggable
4173                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4174             } elsif ($fast_plug_option->{$opt}) {
4175                 # do nothing
4176             } elsif ($opt =~ m/^net(\d+)$/) {
4177                 die "skip\n" if !$hotplug_features->{network};
4178                 vm_deviceunplug($vmid, $conf, $opt);
4179             } elsif (is_valid_drivename($opt)) {
4180                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4181                 vm_deviceunplug($vmid, $conf, $opt);
4182                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4183             } elsif ($opt =~ m/^memory$/) {
4184                 die "skip\n" if !$hotplug_features->{memory};
4185                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4186             } elsif ($opt eq 'cpuunits') {
4187                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4188             } elsif ($opt eq 'cpulimit') {
4189                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4190             } else {
4191                 die "skip\n";
4192             }
4193         };
4194         if (my $err = $@) {
4195             &$add_error($opt, $err) if $err ne "skip\n";
4196         } else {
4197             # save new config if hotplug was successful
4198             delete $conf->{$opt};
4199             vmconfig_undelete_pending_option($conf, $opt);
4200             PVE::QemuConfig->write_config($vmid, $conf);
4201             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4202         }
4203     }
4204
4205     foreach my $opt (keys %{$conf->{pending}}) {
4206         next if $selection && !$selection->{$opt};
4207         my $value = $conf->{pending}->{$opt};
4208         eval {
4209             if ($opt eq 'hotplug') {
4210                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4211             } elsif ($opt eq 'tablet') {
4212                 die "skip\n" if !$hotplug_features->{usb};
4213                 if ($value == 1) {
4214                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4215                 } elsif ($value == 0) {
4216                     vm_deviceunplug($vmid, $conf, $opt);
4217                 }
4218             } elsif ($opt =~ m/^usb\d+$/) {
4219                 die "skip\n";
4220                 # since we cannot reliably hot unplug usb devices
4221                 # we are disabling it
4222                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4223                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4224                 die "skip\n" if !$d;
4225                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4226             } elsif ($opt eq 'vcpus') {
4227                 die "skip\n" if !$hotplug_features->{cpu};
4228                 qemu_cpu_hotplug($vmid, $conf, $value);
4229             } elsif ($opt eq 'balloon') {
4230                 # enable/disable balloning device is not hotpluggable
4231                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4232                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4233                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4234
4235                 # allow manual ballooning if shares is set to zero
4236                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4237                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4238                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4239                 }
4240             } elsif ($opt =~ m/^net(\d+)$/) {
4241                 # some changes can be done without hotplug
4242                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4243                                     $vmid, $opt, $value);
4244             } elsif (is_valid_drivename($opt)) {
4245                 # some changes can be done without hotplug
4246                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4247                                      $vmid, $opt, $value, 1);
4248             } elsif ($opt =~ m/^memory$/) { #dimms
4249                 die "skip\n" if !$hotplug_features->{memory};
4250                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4251             } elsif ($opt eq 'cpuunits') {
4252                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4253             } elsif ($opt eq 'cpulimit') {
4254                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4255                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4256             } else {
4257                 die "skip\n";  # skip non-hot-pluggable options
4258             }
4259         };
4260         if (my $err = $@) {
4261             &$add_error($opt, $err) if $err ne "skip\n";
4262         } else {
4263             # save new config if hotplug was successful
4264             $conf->{$opt} = $value;
4265             delete $conf->{pending}->{$opt};
4266             PVE::QemuConfig->write_config($vmid, $conf);
4267             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4268         }
4269     }
4270 }
4271
4272 sub try_deallocate_drive {
4273     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4274
4275     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4276         my $volid = $drive->{file};
4277         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4278             my $sid = PVE::Storage::parse_volume_id($volid);
4279             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4280
4281             # check if the disk is really unused
4282             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4283                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4284             PVE::Storage::vdisk_free($storecfg, $volid);
4285             return 1;
4286         } else {
4287             # If vm is not owner of this disk remove from config
4288             return 1;
4289         }
4290     }
4291
4292     return undef;
4293 }
4294
4295 sub vmconfig_delete_or_detach_drive {
4296     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4297
4298     my $drive = parse_drive($opt, $conf->{$opt});
4299
4300     my $rpcenv = PVE::RPCEnvironment::get();
4301     my $authuser = $rpcenv->get_user();
4302
4303     if ($force) {
4304         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4305         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4306     } else {
4307         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4308     }
4309 }
4310
4311 sub vmconfig_apply_pending {
4312     my ($vmid, $conf, $storecfg) = @_;
4313
4314     # cold plug
4315
4316     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4317     while (my ($opt, $force) = each %$pending_delete_hash) {
4318         die "internal error" if $opt =~ m/^unused/;
4319         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4320         if (!defined($conf->{$opt})) {
4321             vmconfig_undelete_pending_option($conf, $opt);
4322             PVE::QemuConfig->write_config($vmid, $conf);
4323         } elsif (is_valid_drivename($opt)) {
4324             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4325             vmconfig_undelete_pending_option($conf, $opt);
4326             delete $conf->{$opt};
4327             PVE::QemuConfig->write_config($vmid, $conf);
4328         } else {
4329             vmconfig_undelete_pending_option($conf, $opt);
4330             delete $conf->{$opt};
4331             PVE::QemuConfig->write_config($vmid, $conf);
4332         }
4333     }
4334
4335     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4336
4337     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4338         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4339
4340         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4341             # skip if nothing changed
4342         } elsif (is_valid_drivename($opt)) {
4343             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4344                 if defined($conf->{$opt});
4345             $conf->{$opt} = $conf->{pending}->{$opt};
4346         } else {
4347             $conf->{$opt} = $conf->{pending}->{$opt};
4348         }
4349
4350         delete $conf->{pending}->{$opt};
4351         PVE::QemuConfig->write_config($vmid, $conf);
4352     }
4353 }
4354
4355 my $safe_num_ne = sub {
4356     my ($a, $b) = @_;
4357
4358     return 0 if !defined($a) && !defined($b);
4359     return 1 if !defined($a);
4360     return 1 if !defined($b);
4361
4362     return $a != $b;
4363 };
4364
4365 my $safe_string_ne = sub {
4366     my ($a, $b) = @_;
4367
4368     return 0 if !defined($a) && !defined($b);
4369     return 1 if !defined($a);
4370     return 1 if !defined($b);
4371
4372     return $a ne $b;
4373 };
4374
4375 sub vmconfig_update_net {
4376     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4377
4378     my $newnet = parse_net($value);
4379
4380     if ($conf->{$opt}) {
4381         my $oldnet = parse_net($conf->{$opt});
4382
4383         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4384             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4385             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4386             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4387
4388             # for non online change, we try to hot-unplug
4389             die "skip\n" if !$hotplug;
4390             vm_deviceunplug($vmid, $conf, $opt);
4391         } else {
4392
4393             die "internal error" if $opt !~ m/net(\d+)/;
4394             my $iface = "tap${vmid}i$1";
4395
4396             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4397                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4398                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4399                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4400                 PVE::Network::tap_unplug($iface);
4401                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4402             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4403                 # Rate can be applied on its own but any change above needs to
4404                 # include the rate in tap_plug since OVS resets everything.
4405                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4406             }
4407
4408             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4409                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4410             }
4411
4412             return 1;
4413         }
4414     }
4415
4416     if ($hotplug) {
4417         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4418     } else {
4419         die "skip\n";
4420     }
4421 }
4422
4423 sub vmconfig_update_disk {
4424     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4425
4426     # fixme: do we need force?
4427
4428     my $drive = parse_drive($opt, $value);
4429
4430     if ($conf->{$opt}) {
4431
4432         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4433
4434             my $media = $drive->{media} || 'disk';
4435             my $oldmedia = $old_drive->{media} || 'disk';
4436             die "unable to change media type\n" if $media ne $oldmedia;
4437
4438             if (!drive_is_cdrom($old_drive)) {
4439
4440                 if ($drive->{file} ne $old_drive->{file}) {
4441
4442                     die "skip\n" if !$hotplug;
4443
4444                     # unplug and register as unused
4445                     vm_deviceunplug($vmid, $conf, $opt);
4446                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4447
4448                 } else {
4449                     # update existing disk
4450
4451                     # skip non hotpluggable value
4452                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4453                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4454                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4455                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4456                         die "skip\n";
4457                     }
4458
4459                     # apply throttle
4460                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4461                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4462                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4463                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4464                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4465                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4466                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4467                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4468                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4469                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4470                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4471                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4472                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4473                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4474                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4475                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4476                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4477                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4478
4479                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4480                                                    ($drive->{mbps} || 0)*1024*1024,
4481                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4482                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4483                                                    $drive->{iops} || 0,
4484                                                    $drive->{iops_rd} || 0,
4485                                                    $drive->{iops_wr} || 0,
4486                                                    ($drive->{mbps_max} || 0)*1024*1024,
4487                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4488                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4489                                                    $drive->{iops_max} || 0,
4490                                                    $drive->{iops_rd_max} || 0,
4491                                                    $drive->{iops_wr_max} || 0,
4492                                                    $drive->{bps_max_length} || 1,
4493                                                    $drive->{bps_rd_max_length} || 1,
4494                                                    $drive->{bps_wr_max_length} || 1,
4495                                                    $drive->{iops_max_length} || 1,
4496                                                    $drive->{iops_rd_max_length} || 1,
4497                                                    $drive->{iops_wr_max_length} || 1);
4498
4499                     }
4500
4501                     return 1;
4502                 }
4503
4504             } else { # cdrom
4505
4506                 if ($drive->{file} eq 'none') {
4507                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4508                 } else {
4509                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4510                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4511                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4512                 }
4513
4514                 return 1;
4515             }
4516         }
4517     }
4518
4519     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4520     # hotplug new disks
4521     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4522     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4523 }
4524
4525 sub vm_start {
4526     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4527         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4528
4529     PVE::QemuConfig->lock_config($vmid, sub {
4530         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4531
4532         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4533
4534         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4535
4536         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4537
4538         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4539             vmconfig_apply_pending($vmid, $conf, $storecfg);
4540             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4541         }
4542
4543         my $defaults = load_defaults();
4544
4545         # set environment variable useful inside network script
4546         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4547
4548         my $local_volumes = {};
4549
4550         if ($targetstorage) {
4551             foreach_drive($conf, sub {
4552                 my ($ds, $drive) = @_;
4553
4554                 return if drive_is_cdrom($drive);
4555
4556                 my $volid = $drive->{file};
4557
4558                 return if !$volid;
4559
4560                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4561
4562                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4563                 return if $scfg->{shared};
4564                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4565             });
4566
4567             my $format = undef;
4568
4569             foreach my $opt (sort keys %$local_volumes) {
4570
4571                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4572                 my $drive = parse_drive($opt, $conf->{$opt});
4573
4574                 #if remote storage is specified, use default format
4575                 if ($targetstorage && $targetstorage ne "1") {
4576                     $storeid = $targetstorage;
4577                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4578                     $format = $defFormat;
4579                 } else {
4580                     #else we use same format than original
4581                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4582                     $format = qemu_img_format($scfg, $volid);
4583                 }
4584
4585                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4586                 my $newdrive = $drive;
4587                 $newdrive->{format} = $format;
4588                 $newdrive->{file} = $newvolid;
4589                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4590                 $local_volumes->{$opt} = $drivestr;
4591                 #pass drive to conf for command line
4592                 $conf->{$opt} = $drivestr;
4593             }
4594         }
4595
4596         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4597
4598         my $migrate_port = 0;
4599         my $migrate_uri;
4600         if ($statefile) {
4601             if ($statefile eq 'tcp') {
4602                 my $localip = "localhost";
4603                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4604                 my $nodename = PVE::INotify::nodename();
4605
4606                 if (!defined($migration_type)) {
4607                     if (defined($datacenterconf->{migration}->{type})) {
4608                         $migration_type = $datacenterconf->{migration}->{type};
4609                     } else {
4610                         $migration_type = 'secure';
4611                     }
4612                 }
4613
4614                 if ($migration_type eq 'insecure') {
4615                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4616                     if ($migrate_network_addr) {
4617                         $localip = $migrate_network_addr;
4618                     } else {
4619                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4620                     }
4621
4622                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4623                 }
4624
4625                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4626                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4627                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4628                 push @$cmd, '-incoming', $migrate_uri;
4629                 push @$cmd, '-S';
4630
4631             } elsif ($statefile eq 'unix') {
4632                 # should be default for secure migrations as a ssh TCP forward
4633                 # tunnel is not deterministic reliable ready and fails regurarly
4634                 # to set up in time, so use UNIX socket forwards
4635                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4636                 unlink $socket_addr;
4637
4638                 $migrate_uri = "unix:$socket_addr";
4639
4640                 push @$cmd, '-incoming', $migrate_uri;
4641                 push @$cmd, '-S';
4642
4643             } else {
4644                 push @$cmd, '-loadstate', $statefile;
4645             }
4646         } elsif ($paused) {
4647             push @$cmd, '-S';
4648         }
4649
4650         # host pci devices
4651         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4652           my $d = parse_hostpci($conf->{"hostpci$i"});
4653           next if !$d;
4654           my $pcidevices = $d->{pciid};
4655           foreach my $pcidevice (@$pcidevices) {
4656                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4657
4658                 my $info = pci_device_info("0000:$pciid");
4659                 die "IOMMU not present\n" if !check_iommu_support();
4660                 die "no pci device info for device '$pciid'\n" if !$info;
4661                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4662                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4663           }
4664         }
4665
4666         PVE::Storage::activate_volumes($storecfg, $vollist);
4667
4668         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4669             my $cmd = [];
4670             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4671             eval  { run_command($cmd); };
4672         }
4673
4674         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4675                                                   : $defaults->{cpuunits};
4676
4677         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4678
4679         my %properties = (
4680             Slice => 'qemu.slice',
4681             KillMode => 'none',
4682             CPUShares => $cpuunits
4683         );
4684
4685         if (my $cpulimit = $conf->{cpulimit}) {
4686             $properties{CPUQuota} = int($cpulimit * 100);
4687         }
4688         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4689
4690         if ($conf->{hugepages}) {
4691
4692             my $code = sub {
4693                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4694                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4695
4696                 PVE::QemuServer::Memory::hugepages_mount();
4697                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4698
4699                 eval  {
4700                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4701                     run_command($cmd, %run_params);
4702                 };
4703
4704                 if (my $err = $@) {
4705                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4706                     die $err;
4707                 }
4708
4709                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4710             };
4711             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4712
4713         } else {
4714             eval  {
4715                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4716                 run_command($cmd, %run_params);
4717             };
4718         }
4719
4720         if (my $err = $@) {
4721             # deactivate volumes if start fails
4722             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4723             die "start failed: $err";
4724         }
4725
4726         print "migration listens on $migrate_uri\n" if $migrate_uri;
4727
4728         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4729             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4730             warn $@ if $@;
4731         }
4732
4733         #start nbd server for storage migration
4734         if ($targetstorage) {
4735             my $nodename = PVE::INotify::nodename();
4736             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4737             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4738             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4739             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4740
4741             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4742
4743             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4744
4745             foreach my $opt (sort keys %$local_volumes) {
4746                 my $volid = $local_volumes->{$opt};
4747                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4748                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4749                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4750             }
4751         }
4752
4753         if ($migratedfrom) {
4754             eval {
4755                 set_migration_caps($vmid);
4756             };
4757             warn $@ if $@;
4758
4759             if ($spice_port) {
4760                 print "spice listens on port $spice_port\n";
4761                 if ($spice_ticket) {
4762                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4763                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4764                 }
4765             }
4766
4767         } else {
4768             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4769                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4770                     if $conf->{balloon};
4771             }
4772
4773             foreach my $opt (keys %$conf) {
4774                 next if $opt !~  m/^net\d+$/;
4775                 my $nicconf = parse_net($conf->{$opt});
4776                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4777             }
4778         }
4779
4780         vm_mon_cmd_nocheck($vmid, 'qom-set',
4781                     path => "machine/peripheral/balloon0",
4782                     property => "guest-stats-polling-interval",
4783                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4784
4785     });
4786 }
4787
4788 sub vm_mon_cmd {
4789     my ($vmid, $execute, %params) = @_;
4790
4791     my $cmd = { execute => $execute, arguments => \%params };
4792     vm_qmp_command($vmid, $cmd);
4793 }
4794
4795 sub vm_mon_cmd_nocheck {
4796     my ($vmid, $execute, %params) = @_;
4797
4798     my $cmd = { execute => $execute, arguments => \%params };
4799     vm_qmp_command($vmid, $cmd, 1);
4800 }
4801
4802 sub vm_qmp_command {
4803     my ($vmid, $cmd, $nocheck) = @_;
4804
4805     my $res;
4806
4807     my $timeout;
4808     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4809         $timeout = $cmd->{arguments}->{timeout};
4810         delete $cmd->{arguments}->{timeout};
4811     }
4812
4813     eval {
4814         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4815         my $sname = qmp_socket($vmid);
4816         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4817             my $qmpclient = PVE::QMPClient->new();
4818
4819             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4820         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4821             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4822                 if scalar(%{$cmd->{arguments}});
4823             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4824         } else {
4825             die "unable to open monitor socket\n";
4826         }
4827     };
4828     if (my $err = $@) {
4829         syslog("err", "VM $vmid qmp command failed - $err");
4830         die $err;
4831     }
4832
4833     return $res;
4834 }
4835
4836 sub vm_human_monitor_command {
4837     my ($vmid, $cmdline) = @_;
4838
4839     my $res;
4840
4841     my $cmd = {
4842         execute => 'human-monitor-command',
4843         arguments => { 'command-line' => $cmdline},
4844     };
4845
4846     return vm_qmp_command($vmid, $cmd);
4847 }
4848
4849 sub vm_commandline {
4850     my ($storecfg, $vmid) = @_;
4851
4852     my $conf = PVE::QemuConfig->load_config($vmid);
4853
4854     my $defaults = load_defaults();
4855
4856     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4857
4858     return PVE::Tools::cmd2string($cmd);
4859 }
4860
4861 sub vm_reset {
4862     my ($vmid, $skiplock) = @_;
4863
4864     PVE::QemuConfig->lock_config($vmid, sub {
4865
4866         my $conf = PVE::QemuConfig->load_config($vmid);
4867
4868         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4869
4870         vm_mon_cmd($vmid, "system_reset");
4871     });
4872 }
4873
4874 sub get_vm_volumes {
4875     my ($conf) = @_;
4876
4877     my $vollist = [];
4878     foreach_volid($conf, sub {
4879         my ($volid, $attr) = @_;
4880
4881         return if $volid =~ m|^/|;
4882
4883         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4884         return if !$sid;
4885
4886         push @$vollist, $volid;
4887     });
4888
4889     return $vollist;
4890 }
4891
4892 sub vm_stop_cleanup {
4893     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4894
4895     eval {
4896
4897         if (!$keepActive) {
4898             my $vollist = get_vm_volumes($conf);
4899             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4900         }
4901
4902         foreach my $ext (qw(mon qmp pid vnc qga)) {
4903             unlink "/var/run/qemu-server/${vmid}.$ext";
4904         }
4905
4906         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4907     };
4908     warn $@ if $@; # avoid errors - just warn
4909 }
4910
4911 # Note: use $nockeck to skip tests if VM configuration file exists.
4912 # We need that when migration VMs to other nodes (files already moved)
4913 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4914 sub vm_stop {
4915     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4916
4917     $force = 1 if !defined($force) && !$shutdown;
4918
4919     if ($migratedfrom){
4920         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4921         kill 15, $pid if $pid;
4922         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4923         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4924         return;
4925     }
4926
4927     PVE::QemuConfig->lock_config($vmid, sub {
4928
4929         my $pid = check_running($vmid, $nocheck);
4930         return if !$pid;
4931
4932         my $conf;
4933         if (!$nocheck) {
4934             $conf = PVE::QemuConfig->load_config($vmid);
4935             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4936             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4937                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4938                 $timeout = $opts->{down} if $opts->{down};
4939             }
4940         }
4941
4942         $timeout = 60 if !defined($timeout);
4943
4944         eval {
4945             if ($shutdown) {
4946                 if (defined($conf) && $conf->{agent}) {
4947                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4948                 } else {
4949                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4950                 }
4951             } else {
4952                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4953             }
4954         };
4955         my $err = $@;
4956
4957         if (!$err) {
4958             my $count = 0;
4959             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4960                 $count++;
4961                 sleep 1;
4962             }
4963
4964             if ($count >= $timeout) {
4965                 if ($force) {
4966                     warn "VM still running - terminating now with SIGTERM\n";
4967                     kill 15, $pid;
4968                 } else {
4969                     die "VM quit/powerdown failed - got timeout\n";
4970                 }
4971             } else {
4972                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4973                 return;
4974             }
4975         } else {
4976             if ($force) {
4977                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4978                 kill 15, $pid;
4979             } else {
4980                 die "VM quit/powerdown failed\n";
4981             }
4982         }
4983
4984         # wait again
4985         $timeout = 10;
4986
4987         my $count = 0;
4988         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4989             $count++;
4990             sleep 1;
4991         }
4992
4993         if ($count >= $timeout) {
4994             warn "VM still running - terminating now with SIGKILL\n";
4995             kill 9, $pid;
4996             sleep 1;
4997         }
4998
4999         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5000    });
5001 }
5002
5003 sub vm_suspend {
5004     my ($vmid, $skiplock) = @_;
5005
5006     PVE::QemuConfig->lock_config($vmid, sub {
5007
5008         my $conf = PVE::QemuConfig->load_config($vmid);
5009
5010         PVE::QemuConfig->check_lock($conf)
5011             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5012
5013         vm_mon_cmd($vmid, "stop");
5014     });
5015 }
5016
5017 sub vm_resume {
5018     my ($vmid, $skiplock, $nocheck) = @_;
5019
5020     PVE::QemuConfig->lock_config($vmid, sub {
5021
5022         if (!$nocheck) {
5023
5024             my $conf = PVE::QemuConfig->load_config($vmid);
5025
5026             PVE::QemuConfig->check_lock($conf)
5027                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5028
5029             vm_mon_cmd($vmid, "cont");
5030
5031         } else {
5032             vm_mon_cmd_nocheck($vmid, "cont");
5033         }
5034     });
5035 }
5036
5037 sub vm_sendkey {
5038     my ($vmid, $skiplock, $key) = @_;
5039
5040     PVE::QemuConfig->lock_config($vmid, sub {
5041
5042         my $conf = PVE::QemuConfig->load_config($vmid);
5043
5044         # there is no qmp command, so we use the human monitor command
5045         vm_human_monitor_command($vmid, "sendkey $key");
5046     });
5047 }
5048
5049 sub vm_destroy {
5050     my ($storecfg, $vmid, $skiplock) = @_;
5051
5052     PVE::QemuConfig->lock_config($vmid, sub {
5053
5054         my $conf = PVE::QemuConfig->load_config($vmid);
5055
5056         if (!check_running($vmid)) {
5057             destroy_vm($storecfg, $vmid, undef, $skiplock);
5058         } else {
5059             die "VM $vmid is running - destroy failed\n";
5060         }
5061     });
5062 }
5063
5064 # pci helpers
5065
5066 sub file_write {
5067     my ($filename, $buf) = @_;
5068
5069     my $fh = IO::File->new($filename, "w");
5070     return undef if !$fh;
5071
5072     my $res = print $fh $buf;
5073
5074     $fh->close();
5075
5076     return $res;
5077 }
5078
5079 sub pci_device_info {
5080     my ($name) = @_;
5081
5082     my $res;
5083
5084     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5085     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5086
5087     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5088     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5089
5090     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5091     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5092
5093     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5094     return undef if !defined($product) || $product !~ s/^0x//;
5095
5096     $res = {
5097         name => $name,
5098         vendor => $vendor,
5099         product => $product,
5100         domain => $domain,
5101         bus => $bus,
5102         slot => $slot,
5103         func => $func,
5104         irq => $irq,
5105         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5106     };
5107
5108     return $res;
5109 }
5110
5111 sub pci_dev_reset {
5112     my ($dev) = @_;
5113
5114     my $name = $dev->{name};
5115
5116     my $fn = "$pcisysfs/devices/$name/reset";
5117
5118     return file_write($fn, "1");
5119 }
5120
5121 sub pci_dev_bind_to_vfio {
5122     my ($dev) = @_;
5123
5124     my $name = $dev->{name};
5125
5126     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5127
5128     if (!-d $vfio_basedir) {
5129         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5130     }
5131     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5132
5133     my $testdir = "$vfio_basedir/$name";
5134     return 1 if -d $testdir;
5135
5136     my $data = "$dev->{vendor} $dev->{product}";
5137     return undef if !file_write("$vfio_basedir/new_id", $data);
5138
5139     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5140     if (!file_write($fn, $name)) {
5141         return undef if -f $fn;
5142     }
5143
5144     $fn = "$vfio_basedir/bind";
5145     if (! -d $testdir) {
5146         return undef if !file_write($fn, $name);
5147     }
5148
5149     return -d $testdir;
5150 }
5151
5152 sub pci_dev_group_bind_to_vfio {
5153     my ($pciid) = @_;
5154
5155     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5156
5157     if (!-d $vfio_basedir) {
5158         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5159     }
5160     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5161
5162     # get IOMMU group devices
5163     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5164       my @devs = grep /^0000:/, readdir($D);
5165     closedir($D);
5166
5167     foreach my $pciid (@devs) {
5168         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5169
5170         # pci bridges, switches or root ports are not supported
5171         # they have a pci_bus subdirectory so skip them
5172         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5173
5174         my $info = pci_device_info($1);
5175         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5176     }
5177
5178     return 1;
5179 }
5180
5181 # vzdump restore implementaion
5182
5183 sub tar_archive_read_firstfile {
5184     my $archive = shift;
5185
5186     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5187
5188     # try to detect archive type first
5189     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5190         die "unable to open file '$archive'\n";
5191     my $firstfile = <$fh>;
5192     kill 15, $pid;
5193     close $fh;
5194
5195     die "ERROR: archive contaions no data\n" if !$firstfile;
5196     chomp $firstfile;
5197
5198     return $firstfile;
5199 }
5200
5201 sub tar_restore_cleanup {
5202     my ($storecfg, $statfile) = @_;
5203
5204     print STDERR "starting cleanup\n";
5205
5206     if (my $fd = IO::File->new($statfile, "r")) {
5207         while (defined(my $line = <$fd>)) {
5208             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5209                 my $volid = $2;
5210                 eval {
5211                     if ($volid =~ m|^/|) {
5212                         unlink $volid || die 'unlink failed\n';
5213                     } else {
5214                         PVE::Storage::vdisk_free($storecfg, $volid);
5215                     }
5216                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5217                 };
5218                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5219             } else {
5220                 print STDERR "unable to parse line in statfile - $line";
5221             }
5222         }
5223         $fd->close();
5224     }
5225 }
5226
5227 sub restore_archive {
5228     my ($archive, $vmid, $user, $opts) = @_;
5229
5230     my $format = $opts->{format};
5231     my $comp;
5232
5233     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5234         $format = 'tar' if !$format;
5235         $comp = 'gzip';
5236     } elsif ($archive =~ m/\.tar$/) {
5237         $format = 'tar' if !$format;
5238     } elsif ($archive =~ m/.tar.lzo$/) {
5239         $format = 'tar' if !$format;
5240         $comp = 'lzop';
5241     } elsif ($archive =~ m/\.vma$/) {
5242         $format = 'vma' if !$format;
5243     } elsif ($archive =~ m/\.vma\.gz$/) {
5244         $format = 'vma' if !$format;
5245         $comp = 'gzip';
5246     } elsif ($archive =~ m/\.vma\.lzo$/) {
5247         $format = 'vma' if !$format;
5248         $comp = 'lzop';
5249     } else {
5250         $format = 'vma' if !$format; # default
5251     }
5252
5253     # try to detect archive format
5254     if ($format eq 'tar') {
5255         return restore_tar_archive($archive, $vmid, $user, $opts);
5256     } else {
5257         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5258     }
5259 }
5260
5261 sub restore_update_config_line {
5262     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5263
5264     return if $line =~ m/^\#qmdump\#/;
5265     return if $line =~ m/^\#vzdump\#/;
5266     return if $line =~ m/^lock:/;
5267     return if $line =~ m/^unused\d+:/;
5268     return if $line =~ m/^parent:/;
5269     return if $line =~ m/^template:/; # restored VM is never a template
5270
5271     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5272     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5273         # try to convert old 1.X settings
5274         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5275         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5276             my ($model, $macaddr) = split(/\=/, $devconfig);
5277             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5278             my $net = {
5279                 model => $model,
5280                 bridge => "vmbr$ind",
5281                 macaddr => $macaddr,
5282             };
5283             my $netstr = print_net($net);
5284
5285             print $outfd "net$cookie->{netcount}: $netstr\n";
5286             $cookie->{netcount}++;
5287         }
5288     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5289         my ($id, $netstr) = ($1, $2);
5290         my $net = parse_net($netstr);
5291         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5292         $netstr = print_net($net);
5293         print $outfd "$id: $netstr\n";
5294     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5295         my $virtdev = $1;
5296         my $value = $3;
5297         my $di = parse_drive($virtdev, $value);
5298         if (defined($di->{backup}) && !$di->{backup}) {
5299             print $outfd "#$line";
5300         } elsif ($map->{$virtdev}) {
5301             delete $di->{format}; # format can change on restore
5302             $di->{file} = $map->{$virtdev};
5303             $value = print_drive($vmid, $di);
5304             print $outfd "$virtdev: $value\n";
5305         } else {
5306             print $outfd $line;
5307         }
5308     } else {
5309         print $outfd $line;
5310     }
5311 }
5312
5313 sub scan_volids {
5314     my ($cfg, $vmid) = @_;
5315
5316     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5317
5318     my $volid_hash = {};
5319     foreach my $storeid (keys %$info) {
5320         foreach my $item (@{$info->{$storeid}}) {
5321             next if !($item->{volid} && $item->{size});
5322             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5323             $volid_hash->{$item->{volid}} = $item;
5324         }
5325     }
5326
5327     return $volid_hash;
5328 }
5329
5330 sub is_volume_in_use {
5331     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5332
5333     my $path = PVE::Storage::path($storecfg, $volid);
5334
5335     my $scan_config = sub {
5336         my ($cref, $snapname) = @_;
5337
5338         foreach my $key (keys %$cref) {
5339             my $value = $cref->{$key};
5340             if (is_valid_drivename($key)) {
5341                 next if $skip_drive && $key eq $skip_drive;
5342                 my $drive = parse_drive($key, $value);
5343                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5344                 return 1 if $volid eq $drive->{file};
5345                 if ($drive->{file} =~ m!^/!) {
5346                     return 1 if $drive->{file} eq $path;
5347                 } else {
5348                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5349                     next if !$storeid;
5350                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5351                     next if !$scfg;
5352                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5353                 }
5354             }
5355         }
5356
5357         return 0;
5358     };
5359
5360     return 1 if &$scan_config($conf);
5361
5362     undef $skip_drive;
5363
5364     foreach my $snapname (keys %{$conf->{snapshots}}) {
5365         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5366     }
5367
5368     return 0;
5369 }
5370
5371 sub update_disksize {
5372     my ($vmid, $conf, $volid_hash) = @_;
5373
5374     my $changes;
5375
5376     # used and unused disks
5377     my $referenced = {};
5378
5379     # Note: it is allowed to define multiple storages with same path (alias), so
5380     # we need to check both 'volid' and real 'path' (two different volid can point
5381     # to the same path).
5382
5383     my $referencedpath = {};
5384
5385     # update size info
5386     foreach my $opt (keys %$conf) {
5387         if (is_valid_drivename($opt)) {
5388             my $drive = parse_drive($opt, $conf->{$opt});
5389             my $volid = $drive->{file};
5390             next if !$volid;
5391
5392             $referenced->{$volid} = 1;
5393             if ($volid_hash->{$volid} &&
5394                 (my $path = $volid_hash->{$volid}->{path})) {
5395                 $referencedpath->{$path} = 1;
5396             }
5397
5398             next if drive_is_cdrom($drive);
5399             next if !$volid_hash->{$volid};
5400
5401             $drive->{size} = $volid_hash->{$volid}->{size};
5402             my $new = print_drive($vmid, $drive);
5403             if ($new ne $conf->{$opt}) {
5404                 $changes = 1;
5405                 $conf->{$opt} = $new;
5406             }
5407         }
5408     }
5409
5410     # remove 'unusedX' entry if volume is used
5411     foreach my $opt (keys %$conf) {
5412         next if $opt !~ m/^unused\d+$/;
5413         my $volid = $conf->{$opt};
5414         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5415         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5416             $changes = 1;
5417             delete $conf->{$opt};
5418         }
5419
5420         $referenced->{$volid} = 1;
5421         $referencedpath->{$path} = 1 if $path;
5422     }
5423
5424     foreach my $volid (sort keys %$volid_hash) {
5425         next if $volid =~ m/vm-$vmid-state-/;
5426         next if $referenced->{$volid};
5427         my $path = $volid_hash->{$volid}->{path};
5428         next if !$path; # just to be sure
5429         next if $referencedpath->{$path};
5430         $changes = 1;
5431         PVE::QemuConfig->add_unused_volume($conf, $volid);
5432         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5433     }
5434
5435     return $changes;
5436 }
5437
5438 sub rescan {
5439     my ($vmid, $nolock) = @_;
5440
5441     my $cfg = PVE::Storage::config();
5442
5443     my $volid_hash = scan_volids($cfg, $vmid);
5444
5445     my $updatefn =  sub {
5446         my ($vmid) = @_;
5447
5448         my $conf = PVE::QemuConfig->load_config($vmid);
5449
5450         PVE::QemuConfig->check_lock($conf);
5451
5452         my $vm_volids = {};
5453         foreach my $volid (keys %$volid_hash) {
5454             my $info = $volid_hash->{$volid};
5455             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5456         }
5457
5458         my $changes = update_disksize($vmid, $conf, $vm_volids);
5459
5460         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5461     };
5462
5463     if (defined($vmid)) {
5464         if ($nolock) {
5465             &$updatefn($vmid);
5466         } else {
5467             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5468         }
5469     } else {
5470         my $vmlist = config_list();
5471         foreach my $vmid (keys %$vmlist) {
5472             if ($nolock) {
5473                 &$updatefn($vmid);
5474             } else {
5475                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5476             }
5477         }
5478     }
5479 }
5480
5481 sub restore_vma_archive {
5482     my ($archive, $vmid, $user, $opts, $comp) = @_;
5483
5484     my $input = $archive eq '-' ? "<&STDIN" : undef;
5485     my $readfrom = $archive;
5486
5487     my $uncomp = '';
5488     if ($comp) {
5489         $readfrom = '-';
5490         my $qarchive = PVE::Tools::shellquote($archive);
5491         if ($comp eq 'gzip') {
5492             $uncomp = "zcat $qarchive|";
5493         } elsif ($comp eq 'lzop') {
5494             $uncomp = "lzop -d -c $qarchive|";
5495         } else {
5496             die "unknown compression method '$comp'\n";
5497         }
5498
5499     }
5500
5501     my $tmpdir = "/var/tmp/vzdumptmp$$";
5502     rmtree $tmpdir;
5503
5504     # disable interrupts (always do cleanups)
5505     local $SIG{INT} =
5506         local $SIG{TERM} =
5507         local $SIG{QUIT} =
5508         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5509
5510     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5511     POSIX::mkfifo($mapfifo, 0600);
5512     my $fifofh;
5513
5514     my $openfifo = sub {
5515         open($fifofh, '>', $mapfifo) || die $!;
5516     };
5517
5518     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5519
5520     my $oldtimeout;
5521     my $timeout = 5;
5522
5523     my $devinfo = {};
5524
5525     my $rpcenv = PVE::RPCEnvironment::get();
5526
5527     my $conffile = PVE::QemuConfig->config_file($vmid);
5528     my $tmpfn = "$conffile.$$.tmp";
5529
5530     # Note: $oldconf is undef if VM does not exists
5531     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5532     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5533
5534     my $print_devmap = sub {
5535         my $virtdev_hash = {};
5536
5537         my $cfgfn = "$tmpdir/qemu-server.conf";
5538
5539         # we can read the config - that is already extracted
5540         my $fh = IO::File->new($cfgfn, "r") ||
5541             "unable to read qemu-server.conf - $!\n";
5542
5543         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5544         if (-f $fwcfgfn) {
5545             my $pve_firewall_dir = '/etc/pve/firewall';
5546             mkdir $pve_firewall_dir; # make sure the dir exists
5547             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5548         }
5549
5550         while (defined(my $line = <$fh>)) {
5551             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5552                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5553                 die "archive does not contain data for drive '$virtdev'\n"
5554                     if !$devinfo->{$devname};
5555                 if (defined($opts->{storage})) {
5556                     $storeid = $opts->{storage} || 'local';
5557                 } elsif (!$storeid) {
5558                     $storeid = 'local';
5559                 }
5560                 $format = 'raw' if !$format;
5561                 $devinfo->{$devname}->{devname} = $devname;
5562                 $devinfo->{$devname}->{virtdev} = $virtdev;
5563                 $devinfo->{$devname}->{format} = $format;
5564                 $devinfo->{$devname}->{storeid} = $storeid;
5565
5566                 # check permission on storage
5567                 my $pool = $opts->{pool}; # todo: do we need that?
5568                 if ($user ne 'root@pam') {
5569                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5570                 }
5571
5572                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5573             }
5574         }
5575
5576         foreach my $devname (keys %$devinfo) {
5577             die "found no device mapping information for device '$devname'\n"
5578                 if !$devinfo->{$devname}->{virtdev};
5579         }
5580
5581         my $cfg = PVE::Storage::config();
5582
5583         # create empty/temp config
5584         if ($oldconf) {
5585             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5586             foreach_drive($oldconf, sub {
5587                 my ($ds, $drive) = @_;
5588
5589                 return if drive_is_cdrom($drive);
5590
5591                 my $volid = $drive->{file};
5592
5593                 return if !$volid || $volid =~ m|^/|;
5594
5595                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5596                 return if !$path || !$owner || ($owner != $vmid);
5597
5598                 # Note: only delete disk we want to restore
5599                 # other volumes will become unused
5600                 if ($virtdev_hash->{$ds}) {
5601                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5602                     if (my $err = $@) {
5603                         warn $err;
5604                     }
5605                 }
5606             });
5607
5608             # delete vmstate files
5609             # since after the restore we have no snapshots anymore
5610             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5611                 my $snap = $oldconf->{snapshots}->{$snapname};
5612                 if ($snap->{vmstate}) {
5613                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5614                     if (my $err = $@) {
5615                         warn $err;
5616                     }
5617                 }
5618             }
5619         }
5620
5621         my $map = {};
5622         foreach my $virtdev (sort keys %$virtdev_hash) {
5623             my $d = $virtdev_hash->{$virtdev};
5624             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5625             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5626
5627             # test if requested format is supported
5628             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5629             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5630             $d->{format} = $defFormat if !$supported;
5631
5632             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5633                                                   $d->{format}, undef, $alloc_size);
5634             print STDERR "new volume ID is '$volid'\n";
5635             $d->{volid} = $volid;
5636             my $path = PVE::Storage::path($cfg, $volid);
5637
5638             PVE::Storage::activate_volumes($cfg,[$volid]);
5639
5640             my $write_zeros = 1;
5641             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5642                 $write_zeros = 0;
5643             }
5644
5645             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5646
5647             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5648             $map->{$virtdev} = $volid;
5649         }
5650
5651         $fh->seek(0, 0) || die "seek failed - $!\n";
5652
5653         my $outfd = new IO::File ($tmpfn, "w") ||
5654             die "unable to write config for VM $vmid\n";
5655
5656         my $cookie = { netcount => 0 };
5657         while (defined(my $line = <$fh>)) {
5658             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5659         }
5660
5661         $fh->close();
5662         $outfd->close();
5663     };
5664
5665     eval {
5666         # enable interrupts
5667         local $SIG{INT} =
5668             local $SIG{TERM} =
5669             local $SIG{QUIT} =
5670             local $SIG{HUP} =
5671             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5672         local $SIG{ALRM} = sub { die "got timeout\n"; };
5673
5674         $oldtimeout = alarm($timeout);
5675
5676         my $parser = sub {
5677             my $line = shift;
5678
5679             print "$line\n";
5680
5681             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5682                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5683                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5684             } elsif ($line =~ m/^CTIME: /) {
5685                 # we correctly received the vma config, so we can disable
5686                 # the timeout now for disk allocation (set to 10 minutes, so
5687                 # that we always timeout if something goes wrong)
5688                 alarm(600);
5689                 &$print_devmap();
5690                 print $fifofh "done\n";
5691                 my $tmp = $oldtimeout || 0;
5692                 $oldtimeout = undef;
5693                 alarm($tmp);
5694                 close($fifofh);
5695             }
5696         };
5697
5698         print "restore vma archive: $cmd\n";
5699         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5700     };
5701     my $err = $@;
5702
5703     alarm($oldtimeout) if $oldtimeout;
5704
5705     my $vollist = [];
5706     foreach my $devname (keys %$devinfo) {
5707         my $volid = $devinfo->{$devname}->{volid};
5708         push @$vollist, $volid if $volid;
5709     }
5710
5711     my $cfg = PVE::Storage::config();
5712     PVE::Storage::deactivate_volumes($cfg, $vollist);
5713
5714     unlink $mapfifo;
5715
5716     if ($err) {
5717         rmtree $tmpdir;
5718         unlink $tmpfn;
5719
5720         foreach my $devname (keys %$devinfo) {
5721             my $volid = $devinfo->{$devname}->{volid};
5722             next if !$volid;
5723             eval {
5724                 if ($volid =~ m|^/|) {
5725                     unlink $volid || die 'unlink failed\n';
5726                 } else {
5727                     PVE::Storage::vdisk_free($cfg, $volid);
5728                 }
5729                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5730             };
5731             print STDERR "unable to cleanup '$volid' - $@" if $@;
5732         }
5733         die $err;
5734     }
5735
5736     rmtree $tmpdir;
5737
5738     rename($tmpfn, $conffile) ||
5739         die "unable to commit configuration file '$conffile'\n";
5740
5741     PVE::Cluster::cfs_update(); # make sure we read new file
5742
5743     eval { rescan($vmid, 1); };
5744     warn $@ if $@;
5745 }
5746
5747 sub restore_tar_archive {
5748     my ($archive, $vmid, $user, $opts) = @_;
5749
5750     if ($archive ne '-') {
5751         my $firstfile = tar_archive_read_firstfile($archive);
5752         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5753             if $firstfile ne 'qemu-server.conf';
5754     }
5755
5756     my $storecfg = PVE::Storage::config();
5757
5758     # destroy existing data - keep empty config
5759     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5760     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5761
5762     my $tocmd = "/usr/lib/qemu-server/qmextract";
5763
5764     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5765     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5766     $tocmd .= ' --prealloc' if $opts->{prealloc};
5767     $tocmd .= ' --info' if $opts->{info};
5768
5769     # tar option "xf" does not autodetect compression when read from STDIN,
5770     # so we pipe to zcat
5771     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5772         PVE::Tools::shellquote("--to-command=$tocmd");
5773
5774     my $tmpdir = "/var/tmp/vzdumptmp$$";
5775     mkpath $tmpdir;
5776
5777     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5778     local $ENV{VZDUMP_VMID} = $vmid;
5779     local $ENV{VZDUMP_USER} = $user;
5780
5781     my $conffile = PVE::QemuConfig->config_file($vmid);
5782     my $tmpfn = "$conffile.$$.tmp";
5783
5784     # disable interrupts (always do cleanups)
5785     local $SIG{INT} =
5786         local $SIG{TERM} =
5787         local $SIG{QUIT} =
5788         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5789
5790     eval {
5791         # enable interrupts
5792         local $SIG{INT} =
5793             local $SIG{TERM} =
5794             local $SIG{QUIT} =
5795             local $SIG{HUP} =
5796             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5797
5798         if ($archive eq '-') {
5799             print "extracting archive from STDIN\n";
5800             run_command($cmd, input => "<&STDIN");
5801         } else {
5802             print "extracting archive '$archive'\n";
5803             run_command($cmd);
5804         }
5805
5806         return if $opts->{info};
5807
5808         # read new mapping
5809         my $map = {};
5810         my $statfile = "$tmpdir/qmrestore.stat";
5811         if (my $fd = IO::File->new($statfile, "r")) {
5812             while (defined (my $line = <$fd>)) {
5813                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5814                     $map->{$1} = $2 if $1;
5815                 } else {
5816                     print STDERR "unable to parse line in statfile - $line\n";
5817                 }
5818             }
5819             $fd->close();
5820         }
5821
5822         my $confsrc = "$tmpdir/qemu-server.conf";
5823
5824         my $srcfd = new IO::File($confsrc, "r") ||
5825             die "unable to open file '$confsrc'\n";
5826
5827         my $outfd = new IO::File ($tmpfn, "w") ||
5828             die "unable to write config for VM $vmid\n";
5829
5830         my $cookie = { netcount => 0 };
5831         while (defined (my $line = <$srcfd>)) {
5832             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5833         }
5834
5835         $srcfd->close();
5836         $outfd->close();
5837     };
5838     my $err = $@;
5839
5840     if ($err) {
5841
5842         unlink $tmpfn;
5843
5844         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5845
5846         die $err;
5847     }
5848
5849     rmtree $tmpdir;
5850
5851     rename $tmpfn, $conffile ||
5852         die "unable to commit configuration file '$conffile'\n";
5853
5854     PVE::Cluster::cfs_update(); # make sure we read new file
5855
5856     eval { rescan($vmid, 1); };
5857     warn $@ if $@;
5858 };
5859
5860 sub foreach_storage_used_by_vm {
5861     my ($conf, $func) = @_;
5862
5863     my $sidhash = {};
5864
5865     foreach_drive($conf, sub {
5866         my ($ds, $drive) = @_;
5867         return if drive_is_cdrom($drive);
5868
5869         my $volid = $drive->{file};
5870
5871         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5872         $sidhash->{$sid} = $sid if $sid;
5873     });
5874
5875     foreach my $sid (sort keys %$sidhash) {
5876         &$func($sid);
5877     }
5878 }
5879
5880 sub do_snapshots_with_qemu {
5881     my ($storecfg, $volid) = @_;
5882
5883     my $storage_name = PVE::Storage::parse_volume_id($volid);
5884
5885     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5886         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5887         return 1;
5888     }
5889
5890     if ($volid =~ m/\.(qcow2|qed)$/){
5891         return 1;
5892     }
5893
5894     return undef;
5895 }
5896
5897 sub qga_check_running {
5898     my ($vmid) = @_;
5899
5900     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5901     if ($@) {
5902         warn "Qemu Guest Agent is not running - $@";
5903         return 0;
5904     }
5905     return 1;
5906 }
5907
5908 sub template_create {
5909     my ($vmid, $conf, $disk) = @_;
5910
5911     my $storecfg = PVE::Storage::config();
5912
5913     foreach_drive($conf, sub {
5914         my ($ds, $drive) = @_;
5915
5916         return if drive_is_cdrom($drive);
5917         return if $disk && $ds ne $disk;
5918
5919         my $volid = $drive->{file};
5920         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5921
5922         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5923         $drive->{file} = $voliddst;
5924         $conf->{$ds} = print_drive($vmid, $drive);
5925         PVE::QemuConfig->write_config($vmid, $conf);
5926     });
5927 }
5928
5929 sub qemu_img_convert {
5930     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5931
5932     my $storecfg = PVE::Storage::config();
5933     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5934     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5935
5936     if ($src_storeid && $dst_storeid) {
5937
5938         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5939
5940         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5941         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5942
5943         my $src_format = qemu_img_format($src_scfg, $src_volname);
5944         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5945
5946         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5947         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5948
5949         my $cmd = [];
5950         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5951         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5952         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5953         if ($is_zero_initialized) {
5954             push @$cmd, "zeroinit:$dst_path";
5955         } else {
5956             push @$cmd, $dst_path;
5957         }
5958
5959         my $parser = sub {
5960             my $line = shift;
5961             if($line =~ m/\((\S+)\/100\%\)/){
5962                 my $percent = $1;
5963                 my $transferred = int($size * $percent / 100);
5964                 my $remaining = $size - $transferred;
5965
5966                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5967             }
5968
5969         };
5970
5971         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5972         my $err = $@;
5973         die "copy failed: $err" if $err;
5974     }
5975 }
5976
5977 sub qemu_img_format {
5978     my ($scfg, $volname) = @_;
5979
5980     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5981         return $1;
5982     } else {
5983         return "raw";
5984     }
5985 }
5986
5987 sub qemu_drive_mirror {
5988     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
5989
5990     $jobs = {} if !$jobs;
5991
5992     my $qemu_target;
5993     my $format;
5994     $jobs->{"drive-$drive"} = {};
5995
5996     if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
5997         my $server = $1;
5998         my $port = $2;
5999         my $exportname = $3;
6000
6001         $format = "nbd";
6002         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
6003         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
6004         my $cmd = ['socat', '-T30', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=5"];
6005
6006         my $pid = fork();
6007         if (!defined($pid)) {
6008             die "forking socat tunnel failed\n";
6009         } elsif ($pid == 0) {
6010             exec(@$cmd);
6011             warn "exec failed: $!\n";
6012             POSIX::_exit(-1);
6013         }
6014         $jobs->{"drive-$drive"}->{pid} = $pid;
6015
6016         my $timeout = 0;
6017         while (!-S $unixsocket) {
6018             die "nbd connection helper timed out\n"
6019                 if $timeout++ > 5;
6020             sleep 1;
6021         }
6022     } else {
6023         my $storecfg = PVE::Storage::config();
6024         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6025
6026         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6027
6028         $format = qemu_img_format($dst_scfg, $dst_volname);
6029
6030         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6031
6032         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6033     }
6034
6035     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6036     $opts->{format} = $format if $format;
6037
6038     print "drive mirror is starting for drive-$drive\n";
6039
6040     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6041
6042     if (my $err = $@) {
6043         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6044         die "mirroring error: $err";
6045     }
6046
6047     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6048 }
6049
6050 sub qemu_drive_mirror_monitor {
6051     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6052
6053     eval {
6054         my $err_complete = 0;
6055
6056         while (1) {
6057             die "storage migration timed out\n" if $err_complete > 300;
6058
6059             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6060
6061             my $running_mirror_jobs = {};
6062             foreach my $stat (@$stats) {
6063                 next if $stat->{type} ne 'mirror';
6064                 $running_mirror_jobs->{$stat->{device}} = $stat;
6065             }
6066
6067             my $readycounter = 0;
6068
6069             foreach my $job (keys %$jobs) {
6070
6071                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6072                     print "$job : finished\n";
6073                     delete $jobs->{$job};
6074                     next;
6075                 }
6076
6077                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6078
6079                 my $busy = $running_mirror_jobs->{$job}->{busy};
6080                 my $ready = $running_mirror_jobs->{$job}->{ready};
6081                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6082                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6083                     my $remaining = $total - $transferred;
6084                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6085
6086                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6087                 }
6088
6089                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6090             }
6091
6092             last if scalar(keys %$jobs) == 0;
6093
6094             if ($readycounter == scalar(keys %$jobs)) {
6095                 print "all mirroring jobs are ready \n";
6096                 last if $skipcomplete; #do the complete later
6097
6098                 if ($vmiddst && $vmiddst != $vmid) {
6099                     my $agent_running = $qga && qga_check_running($vmid);
6100                     if ($agent_running) {
6101                         print "freeze filesystem\n";
6102                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6103                     } else {
6104                         print "suspend vm\n";
6105                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6106                     }
6107
6108                     # if we clone a disk for a new target vm, we don't switch the disk
6109                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6110
6111                     if ($agent_running) {
6112                         print "unfreeze filesystem\n";
6113                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6114                     } else {
6115                         print "resume vm\n";
6116                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6117                     }
6118
6119                     last;
6120                 } else {
6121
6122                     foreach my $job (keys %$jobs) {
6123                         # try to switch the disk if source and destination are on the same guest
6124                         print "$job: Completing block job...\n";
6125
6126                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6127                         if ($@ =~ m/cannot be completed/) {
6128                             print "$job: Block job cannot be completed, try again.\n";
6129                             $err_complete++;
6130                         }else {
6131                             print "$job: Completed successfully.\n";
6132                             $jobs->{$job}->{complete} = 1;
6133                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6134                         }
6135                     }
6136                 }
6137             }
6138             sleep 1;
6139         }
6140     };
6141     my $err = $@;
6142
6143     if ($err) {
6144         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6145         die "mirroring error: $err";
6146     }
6147
6148 }
6149
6150 sub qemu_blockjobs_cancel {
6151     my ($vmid, $jobs) = @_;
6152
6153     foreach my $job (keys %$jobs) {
6154         print "$job: Cancelling block job\n";
6155         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6156         $jobs->{$job}->{cancel} = 1;
6157     }
6158
6159     while (1) {
6160         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6161
6162         my $running_jobs = {};
6163         foreach my $stat (@$stats) {
6164             $running_jobs->{$stat->{device}} = $stat;
6165         }
6166
6167         foreach my $job (keys %$jobs) {
6168
6169             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6170                 print "$job: Done.\n";
6171                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6172                 delete $jobs->{$job};
6173             }
6174         }
6175
6176         last if scalar(keys %$jobs) == 0;
6177
6178         sleep 1;
6179     }
6180 }
6181
6182 sub qemu_blockjobs_finish_tunnel {
6183    my ($vmid, $job, $cpid) = @_;
6184
6185    return if !$cpid;
6186
6187    for (my $i = 1; $i < 20; $i++) {
6188         my $waitpid = waitpid($cpid, WNOHANG);
6189         last if (defined($waitpid) && ($waitpid == $cpid));
6190
6191         if ($i == 10) {
6192             kill(15, $cpid);
6193         } elsif ($i >= 15) {
6194             kill(9, $cpid);
6195         }
6196         sleep (1);
6197     }
6198     unlink "/run/qemu-server/$vmid.mirror-$job";
6199 }
6200
6201 sub clone_disk {
6202     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6203         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6204
6205     my $newvolid;
6206
6207     if (!$full) {
6208         print "create linked clone of drive $drivename ($drive->{file})\n";
6209         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6210         push @$newvollist, $newvolid;
6211     } else {
6212
6213         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6214         $storeid = $storage if $storage;
6215
6216         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6217         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6218
6219         print "create full clone of drive $drivename ($drive->{file})\n";
6220         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, undef, ($size/1024));
6221         push @$newvollist, $newvolid;
6222
6223         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6224
6225         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6226         if (!$running || $snapname) {
6227             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6228         } else {
6229
6230             my $kvmver = get_running_qemu_version ($vmid);
6231             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6232                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6233                     if $drive->{iothread};
6234             }
6235
6236             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6237         }
6238     }
6239
6240     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6241
6242     my $disk = $drive;
6243     $disk->{format} = undef;
6244     $disk->{file} = $newvolid;
6245     $disk->{size} = $size;
6246
6247     return $disk;
6248 }
6249
6250 # this only works if VM is running
6251 sub get_current_qemu_machine {
6252     my ($vmid) = @_;
6253
6254     my $cmd = { execute => 'query-machines', arguments => {} };
6255     my $res = vm_qmp_command($vmid, $cmd);
6256
6257     my ($current, $default);
6258     foreach my $e (@$res) {
6259         $default = $e->{name} if $e->{'is-default'};
6260         $current = $e->{name} if $e->{'is-current'};
6261     }
6262
6263     # fallback to the default machine if current is not supported by qemu
6264     return $current || $default || 'pc';
6265 }
6266
6267 sub get_running_qemu_version {
6268     my ($vmid) = @_;
6269     my $cmd = { execute => 'query-version', arguments => {} };
6270     my $res = vm_qmp_command($vmid, $cmd);
6271     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6272 }
6273
6274 sub qemu_machine_feature_enabled {
6275     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6276
6277     my $current_major;
6278     my $current_minor;
6279
6280     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6281
6282         $current_major = $3;
6283         $current_minor = $4;
6284
6285     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6286
6287         $current_major = $1;
6288         $current_minor = $2;
6289     }
6290
6291     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6292
6293
6294 }
6295
6296 sub qemu_machine_pxe {
6297     my ($vmid, $conf, $machine) = @_;
6298
6299     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6300
6301     foreach my $opt (keys %$conf) {
6302         next if $opt !~ m/^net(\d+)$/;
6303         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6304         next if !$net;
6305         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6306         return $machine.".pxe" if $romfile =~ m/pxe/;
6307         last;
6308     }
6309
6310     return $machine;
6311 }
6312
6313 sub qemu_use_old_bios_files {
6314     my ($machine_type) = @_;
6315
6316     return if !$machine_type;
6317
6318     my $use_old_bios_files = undef;
6319
6320     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6321         $machine_type = $1;
6322         $use_old_bios_files = 1;
6323     } else {
6324         my $kvmver = kvm_user_version();
6325         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6326         # load new efi bios files on migration. So this hack is required to allow
6327         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6328         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6329         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6330     }
6331
6332     return ($use_old_bios_files, $machine_type);
6333 }
6334
6335 sub create_efidisk {
6336     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6337
6338     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6339
6340     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6341     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6342     PVE::Storage::activate_volumes($storecfg, [$volid]);
6343
6344     my $path = PVE::Storage::path($storecfg, $volid);
6345     eval {
6346         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6347     };
6348     die "Copying EFI vars image failed: $@" if $@;
6349
6350     return ($volid, $vars_size);
6351 }
6352
6353 sub lspci {
6354
6355     my $devices = {};
6356
6357     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6358             my (undef, $id, $function) = @_;
6359             my $res = { id => $id, function => $function};
6360             push @{$devices->{$id}}, $res;
6361     });
6362
6363     # Entries should be sorted by functions.
6364     foreach my $id (keys %$devices) {
6365         my $dev = $devices->{$id};
6366         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6367     }
6368
6369     return $devices;
6370 }
6371
6372 sub vm_iothreads_list {
6373     my ($vmid) = @_;
6374
6375     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6376
6377     my $iothreads = {};
6378     foreach my $iothread (@$res) {
6379         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6380     }
6381
6382     return $iothreads;
6383 }
6384
6385 sub scsihw_infos {
6386     my ($conf, $drive) = @_;
6387
6388     my $maxdev = 0;
6389
6390     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6391         $maxdev = 7;
6392     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6393         $maxdev = 1;
6394     } else {
6395         $maxdev = 256;
6396     }
6397
6398     my $controller = int($drive->{index} / $maxdev);
6399     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6400
6401     return ($maxdev, $controller, $controller_prefix);
6402 }
6403
6404 sub add_hyperv_enlightenments {
6405     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6406
6407     return if $winversion < 6;
6408     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6409
6410     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6411
6412     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6413         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6414         push @$cpuFlags , 'hv_vapic';
6415         push @$cpuFlags , 'hv_time';
6416     } else {
6417         push @$cpuFlags , 'hv_spinlocks=0xffff';
6418     }
6419
6420     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6421         push @$cpuFlags , 'hv_reset';
6422         push @$cpuFlags , 'hv_vpindex';
6423         push @$cpuFlags , 'hv_runtime';
6424     }
6425
6426     if ($winversion >= 7) {
6427         push @$cpuFlags , 'hv_relaxed';
6428     }
6429 }
6430
6431 sub windows_version {
6432     my ($ostype) = @_;
6433
6434     return 0 if !$ostype;
6435
6436     my $winversion = 0;
6437
6438     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6439         $winversion = 5;
6440     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6441         $winversion = 6;
6442     } elsif ($ostype =~ m/^win(\d+)$/) {
6443         $winversion = $1;
6444     }
6445
6446     return $winversion;
6447 }
6448
6449 sub resolve_dst_disk_format {
6450         my ($storecfg, $storeid, $src_volname, $format) = @_;
6451         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6452
6453         if (!$format) {
6454             # if no target format is specified, use the source disk format as hint
6455             if ($src_volname) {
6456                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6457                 $format = qemu_img_format($scfg, $src_volname);
6458             } else {
6459                 return $defFormat;
6460             }
6461         }
6462
6463         # test if requested format is supported - else use default
6464         my $supported = grep { $_ eq $format } @$validFormats;
6465         $format = $defFormat if !$supported;
6466         return $format;
6467 }
6468
6469 sub resolve_first_disk {
6470     my $conf = shift;
6471     my @disks = PVE::QemuServer::valid_drive_names();
6472     my $firstdisk;
6473     foreach my $ds (reverse @disks) {
6474         next if !$conf->{$ds};
6475         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6476         next if PVE::QemuServer::drive_is_cdrom($disk);
6477         $firstdisk = $ds;
6478     }
6479     return $firstdisk;
6480 }
6481
6482 sub generate_smbios1_uuid {
6483     my ($uuid, $uuid_str);
6484     UUID::generate($uuid);
6485     UUID::unparse($uuid, $uuid_str);
6486     return "uuid=$uuid_str";
6487 }
6488
6489 # bash completion helper
6490
6491 sub complete_backup_archives {
6492     my ($cmdname, $pname, $cvalue) = @_;
6493
6494     my $cfg = PVE::Storage::config();
6495
6496     my $storeid;
6497
6498     if ($cvalue =~ m/^([^:]+):/) {
6499         $storeid = $1;
6500     }
6501
6502     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6503
6504     my $res = [];
6505     foreach my $id (keys %$data) {
6506         foreach my $item (@{$data->{$id}}) {
6507             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6508             push @$res, $item->{volid} if defined($item->{volid});
6509         }
6510     }
6511
6512     return $res;
6513 }
6514
6515 my $complete_vmid_full = sub {
6516     my ($running) = @_;
6517
6518     my $idlist = vmstatus();
6519
6520     my $res = [];
6521
6522     foreach my $id (keys %$idlist) {
6523         my $d = $idlist->{$id};
6524         if (defined($running)) {
6525             next if $d->{template};
6526             next if $running && $d->{status} ne 'running';
6527             next if !$running && $d->{status} eq 'running';
6528         }
6529         push @$res, $id;
6530
6531     }
6532     return $res;
6533 };
6534
6535 sub complete_vmid {
6536     return &$complete_vmid_full();
6537 }
6538
6539 sub complete_vmid_stopped {
6540     return &$complete_vmid_full(0);
6541 }
6542
6543 sub complete_vmid_running {
6544     return &$complete_vmid_full(1);
6545 }
6546
6547 sub complete_storage {
6548
6549     my $cfg = PVE::Storage::config();
6550     my $ids = $cfg->{ids};
6551
6552     my $res = [];
6553     foreach my $sid (keys %$ids) {
6554         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6555         next if !$ids->{$sid}->{content}->{images};
6556         push @$res, $sid;
6557     }
6558
6559     return $res;
6560 }
6561
6562 sub nbd_stop {
6563     my ($vmid) = @_;
6564
6565     vm_mon_cmd($vmid, 'nbd-server-stop');
6566 }
6567
6568 1;