PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42
  43 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  44
  45 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  46
  47 # Note about locking: we use flock on the config file protect
  48 # against concurent actions.
  49 # Aditionaly, we have a 'lock' setting in the config file. This
  50 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  51 # allowed when such lock is set. But you can ignore this kind of
  52 # lock with the --skiplock flag.
  53
  54 cfs_register_file('/qemu-server/',
  55                   \&parse_vm_config,
  56                   \&write_vm_config);
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  72     type => 'string',
  73     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  74     description => "The drive's backing file's data format.",
  75     optional => 1,
  76 });
  77
  78 #no warnings 'redefine';
  79
  80 sub cgroups_write {
  81    my ($controller, $vmid, $option, $value) = @_;
  82
  83    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  84    PVE::ProcFSTools::write_proc_entry($path, $value);
  85
  86 }
  87
  88 my $nodename = PVE::INotify::nodename();
  89
  90 mkdir "/etc/pve/nodes/$nodename";
  91 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  92 mkdir $confdir;
  93
  94 my $var_run_tmpdir = "/var/run/qemu-server";
  95 mkdir $var_run_tmpdir;
  96
  97 my $lock_dir = "/var/lock/qemu-server";
  98 mkdir $lock_dir;
  99
 100 my $pcisysfs = "/sys/bus/pci";
 101
 102 my $cpu_vendor_list = {
 103     # Intel CPUs
 104     486 => 'GenuineIntel',
 105     pentium => 'GenuineIntel',
 106     pentium2  => 'GenuineIntel',
 107     pentium3  => 'GenuineIntel',
 108     coreduo => 'GenuineIntel',
 109     core2duo => 'GenuineIntel',
 110     Conroe  => 'GenuineIntel',
 111     Penryn  => 'GenuineIntel',
 112     Nehalem  => 'GenuineIntel',
 113     Westmere => 'GenuineIntel',
 114     SandyBridge => 'GenuineIntel',
 115     IvyBridge => 'GenuineIntel',
 116     Haswell => 'GenuineIntel',
 117     'Haswell-noTSX' => 'GenuineIntel',
 118     Broadwell => 'GenuineIntel',
 119     'Broadwell-noTSX' => 'GenuineIntel',
 120     'Skylake-Client' => 'GenuineIntel',
 121
 122     # AMD CPUs
 123     athlon => 'AuthenticAMD',
 124     phenom  => 'AuthenticAMD',
 125     Opteron_G1  => 'AuthenticAMD',
 126     Opteron_G2  => 'AuthenticAMD',
 127     Opteron_G3  => 'AuthenticAMD',
 128     Opteron_G4  => 'AuthenticAMD',
 129     Opteron_G5  => 'AuthenticAMD',
 130
 131     # generic types, use vendor from host node
 132     host => 'default',
 133     kvm32 => 'default',
 134     kvm64 => 'default',
 135     qemu32 => 'default',
 136     qemu64 => 'default',
 137 };
 138
 139 my $cpu_fmt = {
 140     cputype => {
 141         description => "Emulated CPU type.",
 142         type => 'string',
 143         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 144         default => 'kvm64',
 145         default_key => 1,
 146     },
 147     hidden => {
 148         description => "Do not identify as a KVM virtual machine.",
 149         type => 'boolean',
 150         optional => 1,
 151         default => 0
 152     },
 153 };
 154
 155 my $watchdog_fmt = {
 156     model => {
 157         default_key => 1,
 158         type => 'string',
 159         enum => [qw(i6300esb ib700)],
 160         description => "Watchdog type to emulate.",
 161         default => 'i6300esb',
 162         optional => 1,
 163     },
 164     action => {
 165         type => 'string',
 166         enum => [qw(reset shutdown poweroff pause debug none)],
 167         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 168         optional => 1,
 169     },
 170 };
 171 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 172
 173 my $confdesc = {
 174     onboot => {
 175         optional => 1,
 176         type => 'boolean',
 177         description => "Specifies whether a VM will be started during system bootup.",
 178         default => 0,
 179     },
 180     autostart => {
 181         optional => 1,
 182         type => 'boolean',
 183         description => "Automatic restart after crash (currently ignored).",
 184         default => 0,
 185     },
 186     hotplug => {
 187         optional => 1,
 188         type => 'string', format => 'pve-hotplug-features',
 189         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 190         default => 'network,disk,usb',
 191     },
 192     reboot => {
 193         optional => 1,
 194         type => 'boolean',
 195         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 196         default => 1,
 197     },
 198     lock => {
 199         optional => 1,
 200         type => 'string',
 201         description => "Lock/unlock the VM.",
 202         enum => [qw(migrate backup snapshot rollback)],
 203     },
 204     cpulimit => {
 205         optional => 1,
 206         type => 'number',
 207         description => "Limit of CPU usage.",
 208         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 209         minimum => 0,
 210         maximum => 128,
 211         default => 0,
 212     },
 213     cpuunits => {
 214         optional => 1,
 215         type => 'integer',
 216         description => "CPU weight for a VM.",
 217         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 218         minimum => 2,
 219         maximum => 262144,
 220         default => 1024,
 221     },
 222     memory => {
 223         optional => 1,
 224         type => 'integer',
 225         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 226         minimum => 16,
 227         default => 512,
 228     },
 229     balloon => {
 230         optional => 1,
 231         type => 'integer',
 232         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 233         minimum => 0,
 234     },
 235     shares => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 239         minimum => 0,
 240         maximum => 50000,
 241         default => 1000,
 242     },
 243     keyboard => {
 244         optional => 1,
 245         type => 'string',
 246         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.".
 247                        "It should not be necessary to set it.",
 248         enum => PVE::Tools::kvmkeymaplist(),
 249         default => undef,
 250     },
 251     name => {
 252         optional => 1,
 253         type => 'string', format => 'dns-name',
 254         description => "Set a name for the VM. Only used on the configuration web interface.",
 255     },
 256     scsihw => {
 257         optional => 1,
 258         type => 'string',
 259         description => "SCSI controller model",
 260         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 261         default => 'lsi',
 262     },
 263     description => {
 264         optional => 1,
 265         type => 'string',
 266         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 267     },
 268     ostype => {
 269         optional => 1,
 270         type => 'string',
 271         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 272         description => "Specify guest operating system.",
 273         verbose_description => <<EODESC,
 274 Specify guest operating system. This is used to enable special
 275 optimization/features for specific operating systems:
 276
 277 [horizontal]
 278 other;; unspecified OS
 279 wxp;; Microsoft Windows XP
 280 w2k;; Microsoft Windows 2000
 281 w2k3;; Microsoft Windows 2003
 282 w2k8;; Microsoft Windows 2008
 283 wvista;; Microsoft Windows Vista
 284 win7;; Microsoft Windows 7
 285 win8;; Microsoft Windows 8/2012/2012r2
 286 win10;; Microsoft Windows 10/2016
 287 l24;; Linux 2.4 Kernel
 288 l26;; Linux 2.6/3.X Kernel
 289 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 290 EODESC
 291     },
 292     boot => {
 293         optional => 1,
 294         type => 'string',
 295         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 296         pattern => '[acdn]{1,4}',
 297         default => 'cdn',
 298     },
 299     bootdisk => {
 300         optional => 1,
 301         type => 'string', format => 'pve-qm-bootdisk',
 302         description => "Enable booting from specified disk.",
 303         pattern => '(ide|sata|scsi|virtio)\d+',
 304     },
 305     smp => {
 306         optional => 1,
 307         type => 'integer',
 308         description => "The number of CPUs. Please use option -sockets instead.",
 309         minimum => 1,
 310         default => 1,
 311     },
 312     sockets => {
 313         optional => 1,
 314         type => 'integer',
 315         description => "The number of CPU sockets.",
 316         minimum => 1,
 317         default => 1,
 318     },
 319     cores => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "The number of cores per socket.",
 323         minimum => 1,
 324         default => 1,
 325     },
 326     numa => {
 327         optional => 1,
 328         type => 'boolean',
 329         description => "Enable/disable NUMA.",
 330         default => 0,
 331     },
 332     hugepages => {
 333         optional => 1,
 334         type => 'string',
 335         description => "Enable/disable hugepages memory.",
 336         enum => [qw(any 2 1024)],
 337     },
 338     vcpus => {
 339         optional => 1,
 340         type => 'integer',
 341         description => "Number of hotplugged vcpus.",
 342         minimum => 1,
 343         default => 0,
 344     },
 345     acpi => {
 346         optional => 1,
 347         type => 'boolean',
 348         description => "Enable/disable ACPI.",
 349         default => 1,
 350     },
 351     agent => {
 352         optional => 1,
 353         type => 'boolean',
 354         description => "Enable/disable Qemu GuestAgent.",
 355         default => 0,
 356     },
 357     kvm => {
 358         optional => 1,
 359         type => 'boolean',
 360         description => "Enable/disable KVM hardware virtualization.",
 361         default => 1,
 362     },
 363     tdf => {
 364         optional => 1,
 365         type => 'boolean',
 366         description => "Enable/disable time drift fix.",
 367         default => 0,
 368     },
 369     localtime => {
 370         optional => 1,
 371         type => 'boolean',
 372         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 373     },
 374     freeze => {
 375         optional => 1,
 376         type => 'boolean',
 377         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 378     },
 379     vga => {
 380         optional => 1,
 381         type => 'string',
 382         description => "Select the VGA type.",
 383         verbose_description => "Select the VGA type. If you want to use high resolution" .
 384             " modes (>= 1280x1024x16) then you should use the options " .
 385             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 386             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 387             "display sever. For win* OS you can select how many independent " .
 388             "displays you want, Linux guests can add displays them self. " .
 389             "You can also run without any graphic card, using a serial device" .
 390             " as terminal.",
 391         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 392     },
 393     watchdog => {
 394         optional => 1,
 395         type => 'string', format => 'pve-qm-watchdog',
 396         description => "Create a virtual hardware watchdog device.",
 397         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 398             " (by a guest action), the watchdog must be periodically polled " .
 399             "by an agent inside the guest or else the watchdog will reset " .
 400             "the guest (or execute the respective action specified)",
 401     },
 402     startdate => {
 403         optional => 1,
 404         type => 'string',
 405         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 406         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 407         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 408         default => 'now',
 409     },
 410     startup =>  get_standard_option('pve-startup-order'),
 411     template => {
 412         optional => 1,
 413         type => 'boolean',
 414         description => "Enable/disable Template.",
 415         default => 0,
 416     },
 417     args => {
 418         optional => 1,
 419         type => 'string',
 420         description => "Arbitrary arguments passed to kvm.",
 421         verbose_description => <<EODESCR,
 422 Arbitrary arguments passed to kvm, for example:
 423
 424 args: -no-reboot -no-hpet
 425
 426 NOTE: this option is for experts only.
 427 EODESCR
 428     },
 429     tablet => {
 430         optional => 1,
 431         type => 'boolean',
 432         default => 1,
 433         description => "Enable/disable the USB tablet device.",
 434         verbose_description => "Enable/disable the USB tablet device. This device is " .
 435             "usually needed to allow absolute mouse positioning with VNC. " .
 436             "Else the mouse runs out of sync with normal VNC clients. " .
 437             "If you're running lots of console-only guests on one host, " .
 438             "you may consider disabling this to save some context switches. " .
 439             "This is turned off by default if you use spice (-vga=qxl).",
 440     },
 441     migrate_speed => {
 442         optional => 1,
 443         type => 'integer',
 444         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 445         minimum => 0,
 446         default => 0,
 447     },
 448     migrate_downtime => {
 449         optional => 1,
 450         type => 'number',
 451         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 452         minimum => 0,
 453         default => 0.1,
 454     },
 455     cdrom => {
 456         optional => 1,
 457         type => 'string', format => 'pve-qm-ide',
 458         typetext => '<volume>',
 459         description => "This is an alias for option -ide2",
 460     },
 461     cpu => {
 462         optional => 1,
 463         description => "Emulated CPU type.",
 464         type => 'string',
 465         format => $cpu_fmt,
 466     },
 467     parent => get_standard_option('pve-snapshot-name', {
 468         optional => 1,
 469         description => "Parent snapshot name. This is used internally, and should not be modified.",
 470     }),
 471     snaptime => {
 472         optional => 1,
 473         description => "Timestamp for snapshots.",
 474         type => 'integer',
 475         minimum => 0,
 476     },
 477     vmstate => {
 478         optional => 1,
 479         type => 'string', format => 'pve-volume-id',
 480         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 481     },
 482     vmstatestorage => get_standard_option('pve-storage-id', {
 483         description => "Default storage for VM state volumes/files.",
 484         optional => 1,
 485     }),
 486     machine => {
 487         description => "Specific the Qemu machine type.",
 488         type => 'string',
 489         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 490         maxLength => 40,
 491         optional => 1,
 492     },
 493     smbios1 => {
 494         description => "Specify SMBIOS type 1 fields.",
 495         type => 'string', format => 'pve-qm-smbios1',
 496         maxLength => 256,
 497         optional => 1,
 498     },
 499     protection => {
 500         optional => 1,
 501         type => 'boolean',
 502         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 503         default => 0,
 504     },
 505     bios => {
 506         optional => 1,
 507         type => 'string',
 508         enum => [ qw(seabios ovmf) ],
 509         description => "Select BIOS implementation.",
 510         default => 'seabios',
 511     },
 512 };
 513
 514 # what about other qemu settings ?
 515 #cpu => 'string',
 516 #machine => 'string',
 517 #fda => 'file',
 518 #fdb => 'file',
 519 #mtdblock => 'file',
 520 #sd => 'file',
 521 #pflash => 'file',
 522 #snapshot => 'bool',
 523 #bootp => 'file',
 524 ##tftp => 'dir',
 525 ##smb => 'dir',
 526 #kernel => 'file',
 527 #append => 'string',
 528 #initrd => 'file',
 529 ##soundhw => 'string',
 530
 531 while (my ($k, $v) = each %$confdesc) {
 532     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 533 }
 534
 535 my $MAX_IDE_DISKS = 4;
 536 my $MAX_SCSI_DISKS = 14;
 537 my $MAX_VIRTIO_DISKS = 16;
 538 my $MAX_SATA_DISKS = 6;
 539 my $MAX_USB_DEVICES = 5;
 540 my $MAX_NETS = 32;
 541 my $MAX_UNUSED_DISKS = 8;
 542 my $MAX_HOSTPCI_DEVICES = 4;
 543 my $MAX_SERIAL_PORTS = 4;
 544 my $MAX_PARALLEL_PORTS = 3;
 545 my $MAX_NUMA = 8;
 546
 547 my $numa_fmt = {
 548     cpus => {
 549         type => "string",
 550         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 551         description => "CPUs accessing this NUMA node.",
 552         format_description => "id[-id];...",
 553     },
 554     memory => {
 555         type => "number",
 556         description => "Amount of memory this NUMA node provides.",
 557         optional => 1,
 558     },
 559     hostnodes => {
 560         type => "string",
 561         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 562         description => "Host NUMA nodes to use.",
 563         format_description => "id[-id];...",
 564         optional => 1,
 565     },
 566     policy => {
 567         type => 'string',
 568         enum => [qw(preferred bind interleave)],
 569         description => "NUMA allocation policy.",
 570         optional => 1,
 571     },
 572 };
 573 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 574 my $numadesc = {
 575     optional => 1,
 576     type => 'string', format => $numa_fmt,
 577     description => "NUMA topology.",
 578 };
 579 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 580
 581 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 582     $confdesc->{"numa$i"} = $numadesc;
 583 }
 584
 585 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 586                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 587                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 588 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 589
 590 my $net_fmt_bridge_descr = <<__EOD__;
 591 Bridge to attach the network device to. The Proxmox VE standard bridge
 592 is called 'vmbr0'.
 593
 594 If you do not specify a bridge, we create a kvm user (NATed) network
 595 device, which provides DHCP and DNS services. The following addresses
 596 are used:
 597
 598  10.0.2.2   Gateway
 599  10.0.2.3   DNS Server
 600  10.0.2.4   SMB Server
 601
 602 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 603 __EOD__
 604
 605 my $net_fmt = {
 606     macaddr => {
 607         type => 'string',
 608         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 609         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 610         format_description => "XX:XX:XX:XX:XX:XX",
 611         optional => 1,
 612     },
 613     model => {
 614         type => 'string',
 615         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 616         enum => $nic_model_list,
 617         default_key => 1,
 618     },
 619     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 620     bridge => {
 621         type => 'string',
 622         description => $net_fmt_bridge_descr,
 623         format_description => 'bridge',
 624         optional => 1,
 625     },
 626     queues => {
 627         type => 'integer',
 628         minimum => 0, maximum => 16,
 629         description => 'Number of packet queues to be used on the device.',
 630         optional => 1,
 631     },
 632     rate => {
 633         type => 'number',
 634         minimum => 0,
 635         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 636         optional => 1,
 637     },
 638     tag => {
 639         type => 'integer',
 640         minimum => 1, maximum => 4094,
 641         description => 'VLAN tag to apply to packets on this interface.',
 642         optional => 1,
 643     },
 644     trunks => {
 645         type => 'string',
 646         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 647         description => 'VLAN trunks to pass through this interface.',
 648         format_description => 'vlanid[;vlanid...]',
 649         optional => 1,
 650     },
 651     firewall => {
 652         type => 'boolean',
 653         description => 'Whether this interface should be protected by the firewall.',
 654         optional => 1,
 655     },
 656     link_down => {
 657         type => 'boolean',
 658         description => 'Whether this interface should be disconnected (like pulling the plug).',
 659         optional => 1,
 660     },
 661 };
 662
 663 my $netdesc = {
 664     optional => 1,
 665     type => 'string', format => $net_fmt,
 666     description => "Specify network devices.",
 667 };
 668
 669 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 670
 671 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 672     $confdesc->{"net$i"} = $netdesc;
 673 }
 674
 675 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 676 sub verify_volume_id_or_qm_path {
 677     my ($volid, $noerr) = @_;
 678
 679     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 680         return $volid;
 681     }
 682
 683     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 684     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 685     if ($@) {
 686         return undef if $noerr;
 687         die $@;
 688     }
 689     return $volid;
 690 }
 691
 692 my $drivename_hash;
 693
 694 my %drivedesc_base = (
 695     volume => { alias => 'file' },
 696     file => {
 697         type => 'string',
 698         format => 'pve-volume-id-or-qm-path',
 699         default_key => 1,
 700         format_description => 'volume',
 701         description => "The drive's backing volume.",
 702     },
 703     media => {
 704         type => 'string',
 705         enum => [qw(cdrom disk)],
 706         description => "The drive's media type.",
 707         default => 'disk',
 708         optional => 1
 709     },
 710     cyls => {
 711         type => 'integer',
 712         description => "Force the drive's physical geometry to have a specific cylinder count.",
 713         optional => 1
 714     },
 715     heads => {
 716         type => 'integer',
 717         description => "Force the drive's physical geometry to have a specific head count.",
 718         optional => 1
 719     },
 720     secs => {
 721         type => 'integer',
 722         description => "Force the drive's physical geometry to have a specific sector count.",
 723         optional => 1
 724     },
 725     trans => {
 726         type => 'string',
 727         enum => [qw(none lba auto)],
 728         description => "Force disk geometry bios translation mode.",
 729         optional => 1,
 730     },
 731     snapshot => {
 732         type => 'boolean',
 733         description => "Whether the drive should be included when making snapshots.",
 734         optional => 1,
 735     },
 736     cache => {
 737         type => 'string',
 738         enum => [qw(none writethrough writeback unsafe directsync)],
 739         description => "The drive's cache mode",
 740         optional => 1,
 741     },
 742     format => get_standard_option('pve-qm-image-format'),
 743     size => {
 744         type => 'string',
 745         format => 'disk-size',
 746         format_description => 'DiskSize',
 747         description => "Disk size. This is purely informational and has no effect.",
 748         optional => 1,
 749     },
 750     backup => {
 751         type => 'boolean',
 752         description => "Whether the drive should be included when making backups.",
 753         optional => 1,
 754     },
 755     replicate => {
 756         type => 'boolean',
 757         description => 'Whether the drive should considered for replication jobs.',
 758         optional => 1,
 759         default => 1,
 760     },
 761     rerror => {
 762         type => 'string',
 763         enum => [qw(ignore report stop)],
 764         description => 'Read error action.',
 765         optional => 1,
 766     },
 767     werror => {
 768         type => 'string',
 769         enum => [qw(enospc ignore report stop)],
 770         description => 'Write error action.',
 771         optional => 1,
 772     },
 773     aio => {
 774         type => 'string',
 775         enum => [qw(native threads)],
 776         description => 'AIO type to use.',
 777         optional => 1,
 778     },
 779     discard => {
 780         type => 'string',
 781         enum => [qw(ignore on)],
 782         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 783         optional => 1,
 784     },
 785     detect_zeroes => {
 786         type => 'boolean',
 787         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 788         optional => 1,
 789     },
 790     serial => {
 791         type => 'string',
 792         format => 'urlencoded',
 793         format_description => 'serial',
 794         maxLength => 20*3, # *3 since it's %xx url enoded
 795         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 796         optional => 1,
 797     }
 798 );
 799
 800 my %iothread_fmt = ( iothread => {
 801         type => 'boolean',
 802         description => "Whether to use iothreads for this drive",
 803         optional => 1,
 804 });
 805
 806 my %model_fmt = (
 807     model => {
 808         type => 'string',
 809         format => 'urlencoded',
 810         format_description => 'model',
 811         maxLength => 40*3, # *3 since it's %xx url enoded
 812         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 813         optional => 1,
 814     },
 815 );
 816
 817 my %queues_fmt = (
 818     queues => {
 819         type => 'integer',
 820         description => "Number of queues.",
 821         minimum => 2,
 822         optional => 1
 823     }
 824 );
 825
 826 my %scsiblock_fmt = (
 827     scsiblock => {
 828         type => 'boolean',
 829         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 830         optional => 1,
 831         default => 0,
 832     },
 833 );
 834
 835 my $add_throttle_desc = sub {
 836     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 837     my $d = {
 838         type => $type,
 839         format_description => $unit,
 840         description => "Maximum $what in $longunit.",
 841         optional => 1,
 842     };
 843     $d->{minimum} = $minimum if defined($minimum);
 844     $drivedesc_base{$key} = $d;
 845 };
 846 # throughput: (leaky bucket)
 847 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 848 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 849 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 850 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 851 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 852 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 853 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 854 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 855 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 856
 857 # pools: (pool of IO before throttling starts taking effect)
 858 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 859 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 860 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 861 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 862 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 863 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 864
 865 # burst lengths
 866 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 867 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 868 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 869 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 870 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 871 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 872
 873 # legacy support
 874 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
 875 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
 876 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
 877 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
 878
 879 my $ide_fmt = {
 880     %drivedesc_base,
 881     %model_fmt,
 882 };
 883 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 884
 885 my $idedesc = {
 886     optional => 1,
 887     type => 'string', format => $ide_fmt,
 888     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 889 };
 890 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 891
 892 my $scsi_fmt = {
 893     %drivedesc_base,
 894     %iothread_fmt,
 895     %queues_fmt,
 896     %scsiblock_fmt,
 897 };
 898 my $scsidesc = {
 899     optional => 1,
 900     type => 'string', format => $scsi_fmt,
 901     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 902 };
 903 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 904
 905 my $sata_fmt = {
 906     %drivedesc_base,
 907 };
 908 my $satadesc = {
 909     optional => 1,
 910     type => 'string', format => $sata_fmt,
 911     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 912 };
 913 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 914
 915 my $virtio_fmt = {
 916     %drivedesc_base,
 917     %iothread_fmt,
 918 };
 919 my $virtiodesc = {
 920     optional => 1,
 921     type => 'string', format => $virtio_fmt,
 922     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 923 };
 924 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 925
 926 my $alldrive_fmt = {
 927     %drivedesc_base,
 928     %iothread_fmt,
 929     %model_fmt,
 930     %queues_fmt,
 931     %scsiblock_fmt,
 932 };
 933
 934 my $efidisk_fmt = {
 935     volume => { alias => 'file' },
 936     file => {
 937         type => 'string',
 938         format => 'pve-volume-id-or-qm-path',
 939         default_key => 1,
 940         format_description => 'volume',
 941         description => "The drive's backing volume.",
 942     },
 943     format => get_standard_option('pve-qm-image-format'),
 944     size => {
 945         type => 'string',
 946         format => 'disk-size',
 947         format_description => 'DiskSize',
 948         description => "Disk size. This is purely informational and has no effect.",
 949         optional => 1,
 950     },
 951 };
 952
 953 my $efidisk_desc = {
 954     optional => 1,
 955     type => 'string', format => $efidisk_fmt,
 956     description => "Configure a Disk for storing EFI vars",
 957 };
 958
 959 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 960
 961 my $usb_fmt = {
 962     host => {
 963         default_key => 1,
 964         type => 'string', format => 'pve-qm-usb-device',
 965         format_description => 'HOSTUSBDEVICE|spice',
 966         description => <<EODESCR,
 967 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 968
 969  'bus-port(.port)*' (decimal numbers) or
 970  'vendor_id:product_id' (hexadeciaml numbers) or
 971  'spice'
 972
 973 You can use the 'lsusb -t' command to list existing usb devices.
 974
 975 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 976
 977 The value 'spice' can be used to add a usb redirection devices for spice.
 978 EODESCR
 979     },
 980     usb3 => {
 981         optional => 1,
 982         type => 'boolean',
 983         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 984         default => 0,
 985     },
 986 };
 987
 988 my $usbdesc = {
 989     optional => 1,
 990     type => 'string', format => $usb_fmt,
 991     description => "Configure an USB device (n is 0 to 4).",
 992 };
 993 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 994
 995 # NOTE: the match-groups of this regex are used in parse_hostpci
 996 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 997 my $hostpci_fmt = {
 998     host => {
 999         default_key => 1,
1000         type => 'string',
1001         pattern => qr/$PCIRE(;$PCIRE)*/,
1002         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1003         description => <<EODESCR,
1004 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1005 of PCI virtual functions of the host. HOSTPCIID syntax is:
1006
1007 'bus:dev.func' (hexadecimal numbers)
1008
1009 You can us the 'lspci' command to list existing PCI devices.
1010 EODESCR
1011     },
1012     rombar => {
1013         type => 'boolean',
1014         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1015         optional => 1,
1016         default => 1,
1017     },
1018     romfile => {
1019         type => 'string',
1020         pattern => '[^,;]+',
1021         format_description => 'string',
1022         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1023         optional => 1,
1024     },
1025     pcie => {
1026         type => 'boolean',
1027         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1028         optional => 1,
1029         default => 0,
1030     },
1031     'x-vga' => {
1032         type => 'boolean',
1033         description =>  "Enable vfio-vga device support.",
1034         optional => 1,
1035         default => 0,
1036     },
1037 };
1038 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1039
1040 my $hostpcidesc = {
1041         optional => 1,
1042         type => 'string', format => 'pve-qm-hostpci',
1043         description => "Map host PCI devices into guest.",
1044         verbose_description =>  <<EODESCR,
1045 Map host PCI devices into guest.
1046
1047 NOTE: This option allows direct access to host hardware. So it is no longer
1048 possible to migrate such machines - use with special care.
1049
1050 CAUTION: Experimental! User reported problems with this option.
1051 EODESCR
1052 };
1053 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1054
1055 my $serialdesc = {
1056         optional => 1,
1057         type => 'string',
1058         pattern => '(/dev/.+|socket)',
1059         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1060         verbose_description =>  <<EODESCR,
1061 Create a serial device inside the VM (n is 0 to 3), and pass through a
1062 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1063 host side (use 'qm terminal' to open a terminal connection).
1064
1065 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1066
1067 CAUTION: Experimental! User reported problems with this option.
1068 EODESCR
1069 };
1070
1071 my $paralleldesc= {
1072         optional => 1,
1073         type => 'string',
1074         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1075         description =>  "Map host parallel devices (n is 0 to 2).",
1076         verbose_description =>  <<EODESCR,
1077 Map host parallel devices (n is 0 to 2).
1078
1079 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1080
1081 CAUTION: Experimental! User reported problems with this option.
1082 EODESCR
1083 };
1084
1085 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1086     $confdesc->{"parallel$i"} = $paralleldesc;
1087 }
1088
1089 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1090     $confdesc->{"serial$i"} = $serialdesc;
1091 }
1092
1093 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1094     $confdesc->{"hostpci$i"} = $hostpcidesc;
1095 }
1096
1097 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1098     $drivename_hash->{"ide$i"} = 1;
1099     $confdesc->{"ide$i"} = $idedesc;
1100 }
1101
1102 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1103     $drivename_hash->{"sata$i"} = 1;
1104     $confdesc->{"sata$i"} = $satadesc;
1105 }
1106
1107 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1108     $drivename_hash->{"scsi$i"} = 1;
1109     $confdesc->{"scsi$i"} = $scsidesc ;
1110 }
1111
1112 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1113     $drivename_hash->{"virtio$i"} = 1;
1114     $confdesc->{"virtio$i"} = $virtiodesc;
1115 }
1116
1117 $drivename_hash->{efidisk0} = 1;
1118 $confdesc->{efidisk0} = $efidisk_desc;
1119
1120 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1121     $confdesc->{"usb$i"} = $usbdesc;
1122 }
1123
1124 my $unuseddesc = {
1125     optional => 1,
1126     type => 'string', format => 'pve-volume-id',
1127     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1128 };
1129
1130 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1131     $confdesc->{"unused$i"} = $unuseddesc;
1132 }
1133
1134 my $kvm_api_version = 0;
1135
1136 sub kvm_version {
1137
1138     return $kvm_api_version if $kvm_api_version;
1139
1140     my $fh = IO::File->new("</dev/kvm") ||
1141         return 0;
1142
1143     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1144         $kvm_api_version = $v;
1145     }
1146
1147     $fh->close();
1148
1149     return  $kvm_api_version;
1150 }
1151
1152 my $kvm_user_version;
1153
1154 sub kvm_user_version {
1155
1156     return $kvm_user_version if $kvm_user_version;
1157
1158     $kvm_user_version = 'unknown';
1159
1160     my $code = sub {
1161         my $line = shift;
1162         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1163             $kvm_user_version = $2;
1164         }
1165     };
1166
1167     eval { run_command("kvm -version", outfunc => $code); };
1168     warn $@ if $@;
1169
1170     return $kvm_user_version;
1171
1172 }
1173
1174 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1175
1176 sub valid_drive_names {
1177     # order is important - used to autoselect boot disk
1178     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1179             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1180             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1181             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1182             'efidisk0');
1183 }
1184
1185 sub is_valid_drivename {
1186     my $dev = shift;
1187
1188     return defined($drivename_hash->{$dev});
1189 }
1190
1191 sub option_exists {
1192     my $key = shift;
1193     return defined($confdesc->{$key});
1194 }
1195
1196 sub nic_models {
1197     return $nic_model_list;
1198 }
1199
1200 sub os_list_description {
1201
1202     return {
1203         other => 'Other',
1204         wxp => 'Windows XP',
1205         w2k => 'Windows 2000',
1206         w2k3 =>, 'Windows 2003',
1207         w2k8 => 'Windows 2008',
1208         wvista => 'Windows Vista',
1209         win7 => 'Windows 7',
1210         win8 => 'Windows 8/2012',
1211         win10 => 'Windows 10/2016',
1212         l24 => 'Linux 2.4',
1213         l26 => 'Linux 2.6',
1214     };
1215 }
1216
1217 my $cdrom_path;
1218
1219 sub get_cdrom_path {
1220
1221     return  $cdrom_path if $cdrom_path;
1222
1223     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1224     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1225     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1226 }
1227
1228 sub get_iso_path {
1229     my ($storecfg, $vmid, $cdrom) = @_;
1230
1231     if ($cdrom eq 'cdrom') {
1232         return get_cdrom_path();
1233     } elsif ($cdrom eq 'none') {
1234         return '';
1235     } elsif ($cdrom =~ m|^/|) {
1236         return $cdrom;
1237     } else {
1238         return PVE::Storage::path($storecfg, $cdrom);
1239     }
1240 }
1241
1242 # try to convert old style file names to volume IDs
1243 sub filename_to_volume_id {
1244     my ($vmid, $file, $media) = @_;
1245
1246     if (!($file eq 'none' || $file eq 'cdrom' ||
1247           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1248
1249         return undef if $file =~ m|/|;
1250
1251         if ($media && $media eq 'cdrom') {
1252             $file = "local:iso/$file";
1253         } else {
1254             $file = "local:$vmid/$file";
1255         }
1256     }
1257
1258     return $file;
1259 }
1260
1261 sub verify_media_type {
1262     my ($opt, $vtype, $media) = @_;
1263
1264     return if !$media;
1265
1266     my $etype;
1267     if ($media eq 'disk') {
1268         $etype = 'images';
1269     } elsif ($media eq 'cdrom') {
1270         $etype = 'iso';
1271     } else {
1272         die "internal error";
1273     }
1274
1275     return if ($vtype eq $etype);
1276
1277     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1278 }
1279
1280 sub cleanup_drive_path {
1281     my ($opt, $storecfg, $drive) = @_;
1282
1283     # try to convert filesystem paths to volume IDs
1284
1285     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1286         ($drive->{file} !~ m|^/dev/.+|) &&
1287         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1288         ($drive->{file} !~ m/^\d+$/)) {
1289         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1290         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1291         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1292         verify_media_type($opt, $vtype, $drive->{media});
1293         $drive->{file} = $volid;
1294     }
1295
1296     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1297 }
1298
1299 sub parse_hotplug_features {
1300     my ($data) = @_;
1301
1302     my $res = {};
1303
1304     return $res if $data eq '0';
1305
1306     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1307
1308     foreach my $feature (PVE::Tools::split_list($data)) {
1309         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1310             $res->{$1} = 1;
1311         } else {
1312             die "invalid hotplug feature '$feature'\n";
1313         }
1314     }
1315     return $res;
1316 }
1317
1318 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1319 sub pve_verify_hotplug_features {
1320     my ($value, $noerr) = @_;
1321
1322     return $value if parse_hotplug_features($value);
1323
1324     return undef if $noerr;
1325
1326     die "unable to parse hotplug option\n";
1327 }
1328
1329 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1330 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1331 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1332 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1333 #        [,iothread=on][,serial=serial][,model=model]
1334
1335 sub parse_drive {
1336     my ($key, $data) = @_;
1337
1338     my ($interface, $index);
1339
1340     if ($key =~ m/^([^\d]+)(\d+)$/) {
1341         $interface = $1;
1342         $index = $2;
1343     } else {
1344         return undef;
1345     }
1346
1347     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1348                                      : $confdesc->{$key}->{format};
1349     if (!$desc) {
1350         warn "invalid drive key: $key\n";
1351         return undef;
1352     }
1353     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1354     return undef if !$res;
1355     $res->{interface} = $interface;
1356     $res->{index} = $index;
1357
1358     my $error = 0;
1359     foreach my $opt (qw(bps bps_rd bps_wr)) {
1360         if (my $bps = defined(delete $res->{$opt})) {
1361             if (defined($res->{"m$opt"})) {
1362                 warn "both $opt and m$opt specified\n";
1363                 ++$error;
1364                 next;
1365             }
1366             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1367         }
1368     }
1369
1370     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1371     for my $requirement (
1372         [mbps_max => 'mbps'],
1373         [mbps_rd_max => 'mbps_rd'],
1374         [mbps_wr_max => 'mbps_wr'],
1375         [miops_max => 'miops'],
1376         [miops_rd_max => 'miops_rd'],
1377         [miops_wr_max => 'miops_wr'],
1378         [bps_max_length => 'mbps_max'],
1379         [bps_rd_max_length => 'mbps_rd_max'],
1380         [bps_wr_max_length => 'mbps_wr_max'],
1381         [iops_max_length => 'iops_max'],
1382         [iops_rd_max_length => 'iops_rd_max'],
1383         [iops_wr_max_length => 'iops_wr_max']) {
1384         my ($option, $requires) = @$requirement;
1385         if ($res->{$option} && !$res->{$requires}) {
1386             warn "$option requires $requires\n";
1387             ++$error;
1388         }
1389     }
1390
1391     return undef if $error;
1392
1393     return undef if $res->{mbps_rd} && $res->{mbps};
1394     return undef if $res->{mbps_wr} && $res->{mbps};
1395     return undef if $res->{iops_rd} && $res->{iops};
1396     return undef if $res->{iops_wr} && $res->{iops};
1397
1398     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1399         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1400         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1401         return undef if $res->{interface} eq 'virtio';
1402     }
1403
1404     if (my $size = $res->{size}) {
1405         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1406     }
1407
1408     return $res;
1409 }
1410
1411 sub print_drive {
1412     my ($vmid, $drive) = @_;
1413     my $data = { %$drive };
1414     delete $data->{$_} for qw(index interface);
1415     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1416 }
1417
1418 sub scsi_inquiry {
1419     my($fh, $noerr) = @_;
1420
1421     my $SG_IO = 0x2285;
1422     my $SG_GET_VERSION_NUM = 0x2282;
1423
1424     my $versionbuf = "\x00" x 8;
1425     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1426     if (!$ret) {
1427         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1428         return undef;
1429     }
1430     my $version = unpack("I", $versionbuf);
1431     if ($version < 30000) {
1432         die "scsi generic interface too old\n"  if !$noerr;
1433         return undef;
1434     }
1435
1436     my $buf = "\x00" x 36;
1437     my $sensebuf = "\x00" x 8;
1438     my $cmd = pack("C x3 C x1", 0x12, 36);
1439
1440     # see /usr/include/scsi/sg.h
1441     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1442
1443     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1444                       length($sensebuf), 0, length($buf), $buf,
1445                       $cmd, $sensebuf, 6000);
1446
1447     $ret = ioctl($fh, $SG_IO, $packet);
1448     if (!$ret) {
1449         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1450         return undef;
1451     }
1452
1453     my @res = unpack($sg_io_hdr_t, $packet);
1454     if ($res[17] || $res[18]) {
1455         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1456         return undef;
1457     }
1458
1459     my $res = {};
1460     (my $byte0, my $byte1, $res->{vendor},
1461      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1462
1463     $res->{removable} = $byte1 & 128 ? 1 : 0;
1464     $res->{type} = $byte0 & 31;
1465
1466     return $res;
1467 }
1468
1469 sub path_is_scsi {
1470     my ($path) = @_;
1471
1472     my $fh = IO::File->new("+<$path") || return undef;
1473     my $res = scsi_inquiry($fh, 1);
1474     close($fh);
1475
1476     return $res;
1477 }
1478
1479 sub machine_type_is_q35 {
1480     my ($conf) = @_;
1481
1482     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1483 }
1484
1485 sub print_tabletdevice_full {
1486     my ($conf) = @_;
1487
1488     my $q35 = machine_type_is_q35($conf);
1489
1490     # we use uhci for old VMs because tablet driver was buggy in older qemu
1491     my $usbbus = $q35 ? "ehci" : "uhci";
1492
1493     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1494 }
1495
1496 sub print_drivedevice_full {
1497     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1498
1499     my $device = '';
1500     my $maxdev = 0;
1501
1502     if ($drive->{interface} eq 'virtio') {
1503         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1504         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1505         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1506     } elsif ($drive->{interface} eq 'scsi') {
1507
1508         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1509         my $unit = $drive->{index} % $maxdev;
1510         my $devicetype = 'hd';
1511         my $path = '';
1512         if (drive_is_cdrom($drive)) {
1513             $devicetype = 'cd';
1514         } else {
1515             if ($drive->{file} =~ m|^/|) {
1516                 $path = $drive->{file};
1517                 if (my $info = path_is_scsi($path)) {
1518                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1519                         $devicetype = 'block';
1520                     } elsif ($info->{type} == 1) { # tape
1521                         $devicetype = 'generic';
1522                     }
1523                 }
1524             } else {
1525                  $path = PVE::Storage::path($storecfg, $drive->{file});
1526             }
1527
1528             if($path =~ m/^iscsi\:\/\//){
1529                 $devicetype = 'generic';
1530             }
1531         }
1532
1533         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1534             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1535         } else {
1536             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1537         }
1538
1539     } elsif ($drive->{interface} eq 'ide'){
1540         $maxdev = 2;
1541         my $controller = int($drive->{index} / $maxdev);
1542         my $unit = $drive->{index} % $maxdev;
1543         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1544
1545         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1546         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1547             $model = URI::Escape::uri_unescape($model);
1548             $device .= ",model=$model";
1549         }
1550     } elsif ($drive->{interface} eq 'sata'){
1551         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1552         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1553         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1554     } elsif ($drive->{interface} eq 'usb') {
1555         die "implement me";
1556         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1557     } else {
1558         die "unsupported interface type";
1559     }
1560
1561     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1562
1563     return $device;
1564 }
1565
1566 sub get_initiator_name {
1567     my $initiator;
1568
1569     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1570     while (defined(my $line = <$fh>)) {
1571         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1572         $initiator = $1;
1573         last;
1574     }
1575     $fh->close();
1576
1577     return $initiator;
1578 }
1579
1580 sub print_drive_full {
1581     my ($storecfg, $vmid, $drive) = @_;
1582
1583     my $path;
1584     my $volid = $drive->{file};
1585     my $format;
1586
1587     if (drive_is_cdrom($drive)) {
1588         $path = get_iso_path($storecfg, $vmid, $volid);
1589     } else {
1590         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1591         if ($storeid) {
1592             $path = PVE::Storage::path($storecfg, $volid);
1593             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1594             $format = qemu_img_format($scfg, $volname);
1595         } else {
1596             $path = $volid;
1597             $format = "raw";
1598         }
1599    }
1600
1601     my $opts = '';
1602     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard);
1603     foreach my $o (@qemu_drive_options) {
1604         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1605     }
1606     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1607         my ($dir, $qmpname) = @$type;
1608         if (my $v = $drive->{"mbps$dir"}) {
1609             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1610         }
1611         if (my $v = $drive->{"mbps${dir}_max"}) {
1612             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1613         }
1614         if (my $v = $drive->{"bps${dir}_max_length"}) {
1615             $opts .= ",throttling.bps$qmpname-max-length=$v";
1616         }
1617         if (my $v = $drive->{"iops${dir}"}) {
1618             $opts .= ",throttling.iops$qmpname=$v";
1619         }
1620         if (my $v = $drive->{"iops${dir}_max"}) {
1621             $opts .= ",throttling.iops$qmpname-max=$v";
1622         }
1623         if (my $v = $drive->{"iops${dir}_max_length"}) {
1624             $opts .= ",throttling.iops$qmpname-max-length=$v";
1625         }
1626     }
1627
1628     if (my $serial = $drive->{serial}) {
1629         $serial = URI::Escape::uri_unescape($serial);
1630         $opts .= ",serial=$serial";
1631     }
1632
1633     $opts .= ",format=$format" if $format && !$drive->{format};
1634
1635     my $cache_direct = 0;
1636
1637     if (my $cache = $drive->{cache}) {
1638         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1639     } elsif (!drive_is_cdrom($drive)) {
1640         $opts .= ",cache=none";
1641         $cache_direct = 1;
1642     }
1643
1644     # aio native works only with O_DIRECT
1645     if (!$drive->{aio}) {
1646         if($cache_direct) {
1647             $opts .= ",aio=native";
1648         } else {
1649             $opts .= ",aio=threads";
1650         }
1651     }
1652
1653     if (!drive_is_cdrom($drive)) {
1654         my $detectzeroes;
1655         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1656             $detectzeroes = 'off';
1657         } elsif ($drive->{discard}) {
1658             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1659         } else {
1660             # This used to be our default with discard not being specified:
1661             $detectzeroes = 'on';
1662         }
1663         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1664     }
1665
1666     my $pathinfo = $path ? "file=$path," : '';
1667
1668     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1669 }
1670
1671 sub print_netdevice_full {
1672     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1673
1674     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1675
1676     my $device = $net->{model};
1677     if ($net->{model} eq 'virtio') {
1678          $device = 'virtio-net-pci';
1679      };
1680
1681     my $pciaddr = print_pci_addr("$netid", $bridges);
1682     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1683     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1684         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1685         my $vectors = $net->{queues} * 2 + 2;
1686         $tmpstr .= ",vectors=$vectors,mq=on";
1687     }
1688     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1689
1690     if ($use_old_bios_files) {
1691         my $romfile;
1692         if ($device eq 'virtio-net-pci') {
1693             $romfile = 'pxe-virtio.rom';
1694         } elsif ($device eq 'e1000') {
1695             $romfile = 'pxe-e1000.rom';
1696         } elsif ($device eq 'ne2k') {
1697             $romfile = 'pxe-ne2k_pci.rom';
1698         } elsif ($device eq 'pcnet') {
1699             $romfile = 'pxe-pcnet.rom';
1700         } elsif ($device eq 'rtl8139') {
1701             $romfile = 'pxe-rtl8139.rom';
1702         }
1703         $tmpstr .= ",romfile=$romfile" if $romfile;
1704     }
1705
1706     return $tmpstr;
1707 }
1708
1709 sub print_netdev_full {
1710     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1711
1712     my $i = '';
1713     if ($netid =~ m/^net(\d+)$/) {
1714         $i = int($1);
1715     }
1716
1717     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1718
1719     my $ifname = "tap${vmid}i$i";
1720
1721     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1722     die "interface name '$ifname' is too long (max 15 character)\n"
1723         if length($ifname) >= 16;
1724
1725     my $vhostparam = '';
1726     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1727
1728     my $vmname = $conf->{name} || "vm$vmid";
1729
1730     my $netdev = "";
1731     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1732
1733     if ($net->{bridge}) {
1734         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1735     } else {
1736         $netdev = "type=user,id=$netid,hostname=$vmname";
1737     }
1738
1739     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1740
1741     return $netdev;
1742 }
1743
1744
1745 sub print_cpu_device {
1746     my ($conf, $id) = @_;
1747
1748     my $kvm = $conf->{kvm} // 1;
1749     my $cpu = $kvm ? "kvm64" : "qemu64";
1750     if (my $cputype = $conf->{cpu}) {
1751         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1752             or die "Cannot parse cpu description: $cputype\n";
1753         $cpu = $cpuconf->{cputype};
1754     }
1755
1756     my $cores = $conf->{cores} || 1;
1757
1758     my $current_core = ($id - 1) % $cores;
1759     my $current_socket = int(($id - 1 - $current_core)/$cores);
1760
1761     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1762 }
1763
1764 sub drive_is_cdrom {
1765     my ($drive) = @_;
1766
1767     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1768
1769 }
1770
1771 sub parse_number_sets {
1772     my ($set) = @_;
1773     my $res = [];
1774     foreach my $part (split(/;/, $set)) {
1775         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1776             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1777             push @$res, [ $1, $2 ];
1778         } else {
1779             die "invalid range: $part\n";
1780         }
1781     }
1782     return $res;
1783 }
1784
1785 sub parse_numa {
1786     my ($data) = @_;
1787
1788     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1789     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1790     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1791     return $res;
1792 }
1793
1794 sub parse_hostpci {
1795     my ($value) = @_;
1796
1797     return undef if !$value;
1798
1799     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1800
1801     my @idlist = split(/;/, $res->{host});
1802     delete $res->{host};
1803     foreach my $id (@idlist) {
1804         if ($id =~ /^$PCIRE$/) {
1805             if (defined($2)) {
1806                 push @{$res->{pciid}}, { id => $1, function => $2 };
1807             } else {
1808                 my $pcidevices = lspci($1);
1809                 $res->{pciid} = $pcidevices->{$1};
1810             }
1811         } else {
1812             # should have been caught by parse_property_string already
1813             die "failed to parse PCI id: $id\n";
1814         }
1815     }
1816     return $res;
1817 }
1818
1819 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1820 sub parse_net {
1821     my ($data) = @_;
1822
1823     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1824     if ($@) {
1825         warn $@;
1826         return undef;
1827     }
1828     if (!defined($res->{macaddr})) {
1829         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1830         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1831     }
1832     return $res;
1833 }
1834
1835 sub print_net {
1836     my $net = shift;
1837
1838     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1839 }
1840
1841 sub add_random_macs {
1842     my ($settings) = @_;
1843
1844     foreach my $opt (keys %$settings) {
1845         next if $opt !~ m/^net(\d+)$/;
1846         my $net = parse_net($settings->{$opt});
1847         next if !$net;
1848         $settings->{$opt} = print_net($net);
1849     }
1850 }
1851
1852 sub vm_is_volid_owner {
1853     my ($storecfg, $vmid, $volid) = @_;
1854
1855     if ($volid !~  m|^/|) {
1856         my ($path, $owner);
1857         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1858         if ($owner && ($owner == $vmid)) {
1859             return 1;
1860         }
1861     }
1862
1863     return undef;
1864 }
1865
1866 sub split_flagged_list {
1867     my $text = shift || '';
1868     $text =~ s/[,;]/ /g;
1869     $text =~ s/^\s+//;
1870     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1871 }
1872
1873 sub join_flagged_list {
1874     my ($how, $lst) = @_;
1875     join $how, map { $lst->{$_} . $_ } keys %$lst;
1876 }
1877
1878 sub vmconfig_delete_pending_option {
1879     my ($conf, $key, $force) = @_;
1880
1881     delete $conf->{pending}->{$key};
1882     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1883     $pending_delete_hash->{$key} = $force ? '!' : '';
1884     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1885 }
1886
1887 sub vmconfig_undelete_pending_option {
1888     my ($conf, $key) = @_;
1889
1890     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1891     delete $pending_delete_hash->{$key};
1892
1893     if (%$pending_delete_hash) {
1894         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1895     } else {
1896         delete $conf->{pending}->{delete};
1897     }
1898 }
1899
1900 sub vmconfig_register_unused_drive {
1901     my ($storecfg, $vmid, $conf, $drive) = @_;
1902
1903     if (!drive_is_cdrom($drive)) {
1904         my $volid = $drive->{file};
1905         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1906             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1907         }
1908     }
1909 }
1910
1911 sub vmconfig_cleanup_pending {
1912     my ($conf) = @_;
1913
1914     # remove pending changes when nothing changed
1915     my $changes;
1916     foreach my $opt (keys %{$conf->{pending}}) {
1917         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1918             $changes = 1;
1919             delete $conf->{pending}->{$opt};
1920         }
1921     }
1922
1923     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1924     my $pending_delete_hash = {};
1925     while (my ($opt, $force) = each %$current_delete_hash) {
1926         if (defined($conf->{$opt})) {
1927             $pending_delete_hash->{$opt} = $force;
1928         } else {
1929             $changes = 1;
1930         }
1931     }
1932
1933     if (%$pending_delete_hash) {
1934         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1935     } else {
1936         delete $conf->{pending}->{delete};
1937     }
1938
1939     return $changes;
1940 }
1941
1942 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1943 my $smbios1_fmt = {
1944     uuid => {
1945         type => 'string',
1946         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1947         format_description => 'UUID',
1948         description => "Set SMBIOS1 UUID.",
1949         optional => 1,
1950     },
1951     version => {
1952         type => 'string',
1953         pattern => '\S+',
1954         format_description => 'string',
1955         description => "Set SMBIOS1 version.",
1956         optional => 1,
1957     },
1958     serial => {
1959         type => 'string',
1960         pattern => '\S+',
1961         format_description => 'string',
1962         description => "Set SMBIOS1 serial number.",
1963         optional => 1,
1964     },
1965     manufacturer => {
1966         type => 'string',
1967         pattern => '\S+',
1968         format_description => 'string',
1969         description => "Set SMBIOS1 manufacturer.",
1970         optional => 1,
1971     },
1972     product => {
1973         type => 'string',
1974         pattern => '\S+',
1975         format_description => 'string',
1976         description => "Set SMBIOS1 product ID.",
1977         optional => 1,
1978     },
1979     sku => {
1980         type => 'string',
1981         pattern => '\S+',
1982         format_description => 'string',
1983         description => "Set SMBIOS1 SKU string.",
1984         optional => 1,
1985     },
1986     family => {
1987         type => 'string',
1988         pattern => '\S+',
1989         format_description => 'string',
1990         description => "Set SMBIOS1 family string.",
1991         optional => 1,
1992     },
1993 };
1994
1995 sub parse_smbios1 {
1996     my ($data) = @_;
1997
1998     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1999     warn $@ if $@;
2000     return $res;
2001 }
2002
2003 sub print_smbios1 {
2004     my ($smbios1) = @_;
2005     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2006 }
2007
2008 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2009
2010 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2011 sub verify_bootdisk {
2012     my ($value, $noerr) = @_;
2013
2014     return $value if is_valid_drivename($value);
2015
2016     return undef if $noerr;
2017
2018     die "invalid boot disk '$value'\n";
2019 }
2020
2021 sub parse_watchdog {
2022     my ($value) = @_;
2023
2024     return undef if !$value;
2025
2026     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2027     warn $@ if $@;
2028     return $res;
2029 }
2030
2031 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2032 sub verify_usb_device {
2033     my ($value, $noerr) = @_;
2034
2035     return $value if parse_usb_device($value);
2036
2037     return undef if $noerr;
2038
2039     die "unable to parse usb device\n";
2040 }
2041
2042 # add JSON properties for create and set function
2043 sub json_config_properties {
2044     my $prop = shift;
2045
2046     foreach my $opt (keys %$confdesc) {
2047         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2048         $prop->{$opt} = $confdesc->{$opt};
2049     }
2050
2051     return $prop;
2052 }
2053
2054 sub check_type {
2055     my ($key, $value) = @_;
2056
2057     die "unknown setting '$key'\n" if !$confdesc->{$key};
2058
2059     my $type = $confdesc->{$key}->{type};
2060
2061     if (!defined($value)) {
2062         die "got undefined value\n";
2063     }
2064
2065     if ($value =~ m/[\n\r]/) {
2066         die "property contains a line feed\n";
2067     }
2068
2069     if ($type eq 'boolean') {
2070         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2071         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2072         die "type check ('boolean') failed - got '$value'\n";
2073     } elsif ($type eq 'integer') {
2074         return int($1) if $value =~ m/^(\d+)$/;
2075         die "type check ('integer') failed - got '$value'\n";
2076     } elsif ($type eq 'number') {
2077         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2078         die "type check ('number') failed - got '$value'\n";
2079     } elsif ($type eq 'string') {
2080         if (my $fmt = $confdesc->{$key}->{format}) {
2081             PVE::JSONSchema::check_format($fmt, $value);
2082             return $value;
2083         }
2084         $value =~ s/^\"(.*)\"$/$1/;
2085         return $value;
2086     } else {
2087         die "internal error"
2088     }
2089 }
2090
2091 sub check_iommu_support{
2092     #fixme : need to check IOMMU support
2093     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2094
2095     my $iommu=1;
2096     return $iommu;
2097
2098 }
2099
2100 sub touch_config {
2101     my ($vmid) = @_;
2102
2103     my $conf = PVE::QemuConfig->config_file($vmid);
2104     utime undef, undef, $conf;
2105 }
2106
2107 sub destroy_vm {
2108     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2109
2110     my $conffile = PVE::QemuConfig->config_file($vmid);
2111
2112     my $conf = PVE::QemuConfig->load_config($vmid);
2113
2114     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2115
2116     if ($conf->{template}) {
2117         # check if any base image is still used by a linked clone
2118         foreach_drive($conf, sub {
2119                 my ($ds, $drive) = @_;
2120
2121                 return if drive_is_cdrom($drive);
2122
2123                 my $volid = $drive->{file};
2124
2125                 return if !$volid || $volid =~ m|^/|;
2126
2127                 die "base volume '$volid' is still in use by linked cloned\n"
2128                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2129
2130         });
2131     }
2132
2133     # only remove disks owned by this VM
2134     foreach_drive($conf, sub {
2135         my ($ds, $drive) = @_;
2136
2137         return if drive_is_cdrom($drive);
2138
2139         my $volid = $drive->{file};
2140
2141         return if !$volid || $volid =~ m|^/|;
2142
2143         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2144         return if !$path || !$owner || ($owner != $vmid);
2145
2146         eval {
2147             PVE::Storage::vdisk_free($storecfg, $volid);
2148         };
2149         warn "Could not remove disk '$volid', check manually: $@" if $@;
2150
2151     });
2152
2153     if ($keep_empty_config) {
2154         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2155     } else {
2156         unlink $conffile;
2157     }
2158
2159     # also remove unused disk
2160     eval {
2161         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2162
2163         eval {
2164             PVE::Storage::foreach_volid($dl, sub {
2165                 my ($volid, $sid, $volname, $d) = @_;
2166                 PVE::Storage::vdisk_free($storecfg, $volid);
2167             });
2168         };
2169         warn $@ if $@;
2170
2171     };
2172     warn $@ if $@;
2173 }
2174
2175 sub parse_vm_config {
2176     my ($filename, $raw) = @_;
2177
2178     return undef if !defined($raw);
2179
2180     my $res = {
2181         digest => Digest::SHA::sha1_hex($raw),
2182         snapshots => {},
2183         pending => {},
2184     };
2185
2186     $filename =~ m|/qemu-server/(\d+)\.conf$|
2187         || die "got strange filename '$filename'";
2188
2189     my $vmid = $1;
2190
2191     my $conf = $res;
2192     my $descr;
2193     my $section = '';
2194
2195     my @lines = split(/\n/, $raw);
2196     foreach my $line (@lines) {
2197         next if $line =~ m/^\s*$/;
2198
2199         if ($line =~ m/^\[PENDING\]\s*$/i) {
2200             $section = 'pending';
2201             if (defined($descr)) {
2202                 $descr =~ s/\s+$//;
2203                 $conf->{description} = $descr;
2204             }
2205             $descr = undef;
2206             $conf = $res->{$section} = {};
2207             next;
2208
2209         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2210             $section = $1;
2211             if (defined($descr)) {
2212                 $descr =~ s/\s+$//;
2213                 $conf->{description} = $descr;
2214             }
2215             $descr = undef;
2216             $conf = $res->{snapshots}->{$section} = {};
2217             next;
2218         }
2219
2220         if ($line =~ m/^\#(.*)\s*$/) {
2221             $descr = '' if !defined($descr);
2222             $descr .= PVE::Tools::decode_text($1) . "\n";
2223             next;
2224         }
2225
2226         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2227             $descr = '' if !defined($descr);
2228             $descr .= PVE::Tools::decode_text($2);
2229         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2230             $conf->{snapstate} = $1;
2231         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2232             my $key = $1;
2233             my $value = $2;
2234             $conf->{$key} = $value;
2235         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2236             my $value = $1;
2237             if ($section eq 'pending') {
2238                 $conf->{delete} = $value; # we parse this later
2239             } else {
2240                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2241             }
2242         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2243             my $key = $1;
2244             my $value = $2;
2245             eval { $value = check_type($key, $value); };
2246             if ($@) {
2247                 warn "vm $vmid - unable to parse value of '$key' - $@";
2248             } else {
2249                 $key = 'ide2' if $key eq 'cdrom';
2250                 my $fmt = $confdesc->{$key}->{format};
2251                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2252                     my $v = parse_drive($key, $value);
2253                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2254                         $v->{file} = $volid;
2255                         $value = print_drive($vmid, $v);
2256                     } else {
2257                         warn "vm $vmid - unable to parse value of '$key'\n";
2258                         next;
2259                     }
2260                 }
2261
2262                 $conf->{$key} = $value;
2263             }
2264         }
2265     }
2266
2267     if (defined($descr)) {
2268         $descr =~ s/\s+$//;
2269         $conf->{description} = $descr;
2270     }
2271     delete $res->{snapstate}; # just to be sure
2272
2273     return $res;
2274 }
2275
2276 sub write_vm_config {
2277     my ($filename, $conf) = @_;
2278
2279     delete $conf->{snapstate}; # just to be sure
2280
2281     if ($conf->{cdrom}) {
2282         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2283         $conf->{ide2} = $conf->{cdrom};
2284         delete $conf->{cdrom};
2285     }
2286
2287     # we do not use 'smp' any longer
2288     if ($conf->{sockets}) {
2289         delete $conf->{smp};
2290     } elsif ($conf->{smp}) {
2291         $conf->{sockets} = $conf->{smp};
2292         delete $conf->{cores};
2293         delete $conf->{smp};
2294     }
2295
2296     my $used_volids = {};
2297
2298     my $cleanup_config = sub {
2299         my ($cref, $pending, $snapname) = @_;
2300
2301         foreach my $key (keys %$cref) {
2302             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2303                 $key eq 'snapstate' || $key eq 'pending';
2304             my $value = $cref->{$key};
2305             if ($key eq 'delete') {
2306                 die "propertry 'delete' is only allowed in [PENDING]\n"
2307                     if !$pending;
2308                 # fixme: check syntax?
2309                 next;
2310             }
2311             eval { $value = check_type($key, $value); };
2312             die "unable to parse value of '$key' - $@" if $@;
2313
2314             $cref->{$key} = $value;
2315
2316             if (!$snapname && is_valid_drivename($key)) {
2317                 my $drive = parse_drive($key, $value);
2318                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2319             }
2320         }
2321     };
2322
2323     &$cleanup_config($conf);
2324
2325     &$cleanup_config($conf->{pending}, 1);
2326
2327     foreach my $snapname (keys %{$conf->{snapshots}}) {
2328         die "internal error" if $snapname eq 'pending';
2329         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2330     }
2331
2332     # remove 'unusedX' settings if we re-add a volume
2333     foreach my $key (keys %$conf) {
2334         my $value = $conf->{$key};
2335         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2336             delete $conf->{$key};
2337         }
2338     }
2339
2340     my $generate_raw_config = sub {
2341         my ($conf, $pending) = @_;
2342
2343         my $raw = '';
2344
2345         # add description as comment to top of file
2346         if (defined(my $descr = $conf->{description})) {
2347             if ($descr) {
2348                 foreach my $cl (split(/\n/, $descr)) {
2349                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2350                 }
2351             } else {
2352                 $raw .= "#\n" if $pending;
2353             }
2354         }
2355
2356         foreach my $key (sort keys %$conf) {
2357             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2358             $raw .= "$key: $conf->{$key}\n";
2359         }
2360         return $raw;
2361     };
2362
2363     my $raw = &$generate_raw_config($conf);
2364
2365     if (scalar(keys %{$conf->{pending}})){
2366         $raw .= "\n[PENDING]\n";
2367         $raw .= &$generate_raw_config($conf->{pending}, 1);
2368     }
2369
2370     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2371         $raw .= "\n[$snapname]\n";
2372         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2373     }
2374
2375     return $raw;
2376 }
2377
2378 sub load_defaults {
2379
2380     my $res = {};
2381
2382     # we use static defaults from our JSON schema configuration
2383     foreach my $key (keys %$confdesc) {
2384         if (defined(my $default = $confdesc->{$key}->{default})) {
2385             $res->{$key} = $default;
2386         }
2387     }
2388
2389     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2390     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2391
2392     return $res;
2393 }
2394
2395 sub config_list {
2396     my $vmlist = PVE::Cluster::get_vmlist();
2397     my $res = {};
2398     return $res if !$vmlist || !$vmlist->{ids};
2399     my $ids = $vmlist->{ids};
2400
2401     foreach my $vmid (keys %$ids) {
2402         my $d = $ids->{$vmid};
2403         next if !$d->{node} || $d->{node} ne $nodename;
2404         next if !$d->{type} || $d->{type} ne 'qemu';
2405         $res->{$vmid}->{exists} = 1;
2406     }
2407     return $res;
2408 }
2409
2410 # test if VM uses local resources (to prevent migration)
2411 sub check_local_resources {
2412     my ($conf, $noerr) = @_;
2413
2414     my $loc_res = 0;
2415
2416     $loc_res = 1 if $conf->{hostusb}; # old syntax
2417     $loc_res = 1 if $conf->{hostpci}; # old syntax
2418
2419     foreach my $k (keys %$conf) {
2420         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2421         # sockets are safe: they will recreated be on the target side post-migrate
2422         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2423         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2424     }
2425
2426     die "VM uses local resources\n" if $loc_res && !$noerr;
2427
2428     return $loc_res;
2429 }
2430
2431 # check if used storages are available on all nodes (use by migrate)
2432 sub check_storage_availability {
2433     my ($storecfg, $conf, $node) = @_;
2434
2435     foreach_drive($conf, sub {
2436         my ($ds, $drive) = @_;
2437
2438         my $volid = $drive->{file};
2439         return if !$volid;
2440
2441         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2442         return if !$sid;
2443
2444         # check if storage is available on both nodes
2445         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2446         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2447    });
2448 }
2449
2450 # list nodes where all VM images are available (used by has_feature API)
2451 sub shared_nodes {
2452     my ($conf, $storecfg) = @_;
2453
2454     my $nodelist = PVE::Cluster::get_nodelist();
2455     my $nodehash = { map { $_ => 1 } @$nodelist };
2456     my $nodename = PVE::INotify::nodename();
2457
2458     foreach_drive($conf, sub {
2459         my ($ds, $drive) = @_;
2460
2461         my $volid = $drive->{file};
2462         return if !$volid;
2463
2464         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2465         if ($storeid) {
2466             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2467             if ($scfg->{disable}) {
2468                 $nodehash = {};
2469             } elsif (my $avail = $scfg->{nodes}) {
2470                 foreach my $node (keys %$nodehash) {
2471                     delete $nodehash->{$node} if !$avail->{$node};
2472                 }
2473             } elsif (!$scfg->{shared}) {
2474                 foreach my $node (keys %$nodehash) {
2475                     delete $nodehash->{$node} if $node ne $nodename
2476                 }
2477             }
2478         }
2479     });
2480
2481     return $nodehash
2482 }
2483
2484 sub check_cmdline {
2485     my ($pidfile, $pid) = @_;
2486
2487     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2488     if (defined($fh)) {
2489         my $line = <$fh>;
2490         $fh->close;
2491         return undef if !$line;
2492         my @param = split(/\0/, $line);
2493
2494         my $cmd = $param[0];
2495         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2496
2497         for (my $i = 0; $i < scalar (@param); $i++) {
2498             my $p = $param[$i];
2499             next if !$p;
2500             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2501                 my $p = $param[$i+1];
2502                 return 1 if $p && ($p eq $pidfile);
2503                 return undef;
2504             }
2505         }
2506     }
2507     return undef;
2508 }
2509
2510 sub check_running {
2511     my ($vmid, $nocheck, $node) = @_;
2512
2513     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2514
2515     die "unable to find configuration file for VM $vmid - no such machine\n"
2516         if !$nocheck && ! -f $filename;
2517
2518     my $pidfile = pidfile_name($vmid);
2519
2520     if (my $fd = IO::File->new("<$pidfile")) {
2521         my $st = stat($fd);
2522         my $line = <$fd>;
2523         close($fd);
2524
2525         my $mtime = $st->mtime;
2526         if ($mtime > time()) {
2527             warn "file '$filename' modified in future\n";
2528         }
2529
2530         if ($line =~ m/^(\d+)$/) {
2531             my $pid = $1;
2532             if (check_cmdline($pidfile, $pid)) {
2533                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2534                     return $pid;
2535                 }
2536             }
2537         }
2538     }
2539
2540     return undef;
2541 }
2542
2543 sub vzlist {
2544
2545     my $vzlist = config_list();
2546
2547     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2548
2549     while (defined(my $de = $fd->read)) {
2550         next if $de !~ m/^(\d+)\.pid$/;
2551         my $vmid = $1;
2552         next if !defined($vzlist->{$vmid});
2553         if (my $pid = check_running($vmid)) {
2554             $vzlist->{$vmid}->{pid} = $pid;
2555         }
2556     }
2557
2558     return $vzlist;
2559 }
2560
2561 sub disksize {
2562     my ($storecfg, $conf) = @_;
2563
2564     my $bootdisk = $conf->{bootdisk};
2565     return undef if !$bootdisk;
2566     return undef if !is_valid_drivename($bootdisk);
2567
2568     return undef if !$conf->{$bootdisk};
2569
2570     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2571     return undef if !defined($drive);
2572
2573     return undef if drive_is_cdrom($drive);
2574
2575     my $volid = $drive->{file};
2576     return undef if !$volid;
2577
2578     return $drive->{size};
2579 }
2580
2581 my $last_proc_pid_stat;
2582
2583 # get VM status information
2584 # This must be fast and should not block ($full == false)
2585 # We only query KVM using QMP if $full == true (this can be slow)
2586 sub vmstatus {
2587     my ($opt_vmid, $full) = @_;
2588
2589     my $res = {};
2590
2591     my $storecfg = PVE::Storage::config();
2592
2593     my $list = vzlist();
2594     my $defaults = load_defaults();
2595
2596     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2597
2598     my $cpucount = $cpuinfo->{cpus} || 1;
2599
2600     foreach my $vmid (keys %$list) {
2601         next if $opt_vmid && ($vmid ne $opt_vmid);
2602
2603         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2604         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2605
2606         my $d = {};
2607         $d->{pid} = $list->{$vmid}->{pid};
2608
2609         # fixme: better status?
2610         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2611
2612         my $size = disksize($storecfg, $conf);
2613         if (defined($size)) {
2614             $d->{disk} = 0; # no info available
2615             $d->{maxdisk} = $size;
2616         } else {
2617             $d->{disk} = 0;
2618             $d->{maxdisk} = 0;
2619         }
2620
2621         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2622             * ($conf->{cores} || $defaults->{cores});
2623         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2624         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2625
2626         $d->{name} = $conf->{name} || "VM $vmid";
2627         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2628             : $defaults->{memory}*(1024*1024);
2629
2630         if ($conf->{balloon}) {
2631             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2632             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2633                 : $defaults->{shares};
2634         }
2635
2636         $d->{uptime} = 0;
2637         $d->{cpu} = 0;
2638         $d->{mem} = 0;
2639
2640         $d->{netout} = 0;
2641         $d->{netin} = 0;
2642
2643         $d->{diskread} = 0;
2644         $d->{diskwrite} = 0;
2645
2646         $d->{template} = PVE::QemuConfig->is_template($conf);
2647
2648         $res->{$vmid} = $d;
2649     }
2650
2651     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2652     foreach my $dev (keys %$netdev) {
2653         next if $dev !~ m/^tap([1-9]\d*)i/;
2654         my $vmid = $1;
2655         my $d = $res->{$vmid};
2656         next if !$d;
2657
2658         $d->{netout} += $netdev->{$dev}->{receive};
2659         $d->{netin} += $netdev->{$dev}->{transmit};
2660
2661         if ($full) {
2662             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2663             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2664         }
2665
2666     }
2667
2668     my $ctime = gettimeofday;
2669
2670     foreach my $vmid (keys %$list) {
2671
2672         my $d = $res->{$vmid};
2673         my $pid = $d->{pid};
2674         next if !$pid;
2675
2676         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2677         next if !$pstat; # not running
2678
2679         my $used = $pstat->{utime} + $pstat->{stime};
2680
2681         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2682
2683         if ($pstat->{vsize}) {
2684             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2685         }
2686
2687         my $old = $last_proc_pid_stat->{$pid};
2688         if (!$old) {
2689             $last_proc_pid_stat->{$pid} = {
2690                 time => $ctime,
2691                 used => $used,
2692                 cpu => 0,
2693             };
2694             next;
2695         }
2696
2697         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2698
2699         if ($dtime > 1000) {
2700             my $dutime = $used -  $old->{used};
2701
2702             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2703             $last_proc_pid_stat->{$pid} = {
2704                 time => $ctime,
2705                 used => $used,
2706                 cpu => $d->{cpu},
2707             };
2708         } else {
2709             $d->{cpu} = $old->{cpu};
2710         }
2711     }
2712
2713     return $res if !$full;
2714
2715     my $qmpclient = PVE::QMPClient->new();
2716
2717     my $ballooncb = sub {
2718         my ($vmid, $resp) = @_;
2719
2720         my $info = $resp->{'return'};
2721         return if !$info->{max_mem};
2722
2723         my $d = $res->{$vmid};
2724
2725         # use memory assigned to VM
2726         $d->{maxmem} = $info->{max_mem};
2727         $d->{balloon} = $info->{actual};
2728
2729         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2730             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2731             $d->{freemem} = $info->{free_mem};
2732         }
2733
2734         $d->{ballooninfo} = $info;
2735     };
2736
2737     my $blockstatscb = sub {
2738         my ($vmid, $resp) = @_;
2739         my $data = $resp->{'return'} || [];
2740         my $totalrdbytes = 0;
2741         my $totalwrbytes = 0;
2742
2743         for my $blockstat (@$data) {
2744             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2745             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2746
2747             $blockstat->{device} =~ s/drive-//;
2748             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2749         }
2750         $res->{$vmid}->{diskread} = $totalrdbytes;
2751         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2752     };
2753
2754     my $statuscb = sub {
2755         my ($vmid, $resp) = @_;
2756
2757         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2758         # this fails if ballon driver is not loaded, so this must be
2759         # the last commnand (following command are aborted if this fails).
2760         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2761
2762         my $status = 'unknown';
2763         if (!defined($status = $resp->{'return'}->{status})) {
2764             warn "unable to get VM status\n";
2765             return;
2766         }
2767
2768         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2769     };
2770
2771     foreach my $vmid (keys %$list) {
2772         next if $opt_vmid && ($vmid ne $opt_vmid);
2773         next if !$res->{$vmid}->{pid}; # not running
2774         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2775     }
2776
2777     $qmpclient->queue_execute(undef, 2);
2778
2779     foreach my $vmid (keys %$list) {
2780         next if $opt_vmid && ($vmid ne $opt_vmid);
2781         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2782     }
2783
2784     return $res;
2785 }
2786
2787 sub foreach_drive {
2788     my ($conf, $func, @param) = @_;
2789
2790     foreach my $ds (valid_drive_names()) {
2791         next if !defined($conf->{$ds});
2792
2793         my $drive = parse_drive($ds, $conf->{$ds});
2794         next if !$drive;
2795
2796         &$func($ds, $drive, @param);
2797     }
2798 }
2799
2800 sub foreach_volid {
2801     my ($conf, $func, @param) = @_;
2802
2803     my $volhash = {};
2804
2805     my $test_volid = sub {
2806         my ($volid, $is_cdrom, $replicate, $snapname) = @_;
2807
2808         return if !$volid;
2809
2810         $volhash->{$volid}->{cdrom} //= 1;
2811         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2812
2813         $volhash->{$volid}->{replicate} //= 0;
2814         $volhash->{$volid}->{replicate} = 1 if $replicate;
2815
2816         $volhash->{$volid}->{referenced_in_config} //= 0;
2817         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
2818
2819         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
2820             if defined($snapname);
2821     };
2822
2823     foreach_drive($conf, sub {
2824         my ($ds, $drive) = @_;
2825         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, undef);
2826     });
2827
2828     foreach my $snapname (keys %{$conf->{snapshots}}) {
2829         my $snap = $conf->{snapshots}->{$snapname};
2830         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
2831         foreach_drive($snap, sub {
2832             my ($ds, $drive) = @_;
2833             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $snapname);
2834         });
2835     }
2836
2837     foreach my $volid (keys %$volhash) {
2838         &$func($volid, $volhash->{$volid}, @param);
2839     }
2840 }
2841
2842 sub vga_conf_has_spice {
2843     my ($vga) = @_;
2844
2845     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2846
2847     return $1 || 1;
2848 }
2849
2850 sub config_to_command {
2851     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2852
2853     my $cmd = [];
2854     my $globalFlags = [];
2855     my $machineFlags = [];
2856     my $rtcFlags = [];
2857     my $cpuFlags = [];
2858     my $devices = [];
2859     my $pciaddr = '';
2860     my $bridges = {};
2861     my $kvmver = kvm_user_version();
2862     my $vernum = 0; # unknown
2863     my $ostype = $conf->{ostype};
2864     my $winversion = windows_version($ostype);
2865     my $kvm = $conf->{kvm} // 1;
2866
2867     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
2868
2869     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2870         $vernum = $1*1000000+$2*1000;
2871     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2872         $vernum = $1*1000000+$2*1000+$3;
2873     }
2874
2875     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2876
2877     my $have_ovz = -f '/proc/vz/vestat';
2878
2879     my $q35 = machine_type_is_q35($conf);
2880     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2881     my $machine_type = $forcemachine || $conf->{machine};
2882     my $use_old_bios_files = undef;
2883     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2884
2885     my $cpuunits = defined($conf->{cpuunits}) ?
2886             $conf->{cpuunits} : $defaults->{cpuunits};
2887
2888     push @$cmd, '/usr/bin/kvm';
2889
2890     push @$cmd, '-id', $vmid;
2891
2892     my $use_virtio = 0;
2893
2894     my $qmpsocket = qmp_socket($vmid);
2895     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2896     push @$cmd, '-mon', "chardev=qmp,mode=control";
2897
2898
2899     push @$cmd, '-pidfile' , pidfile_name($vmid);
2900
2901     push @$cmd, '-daemonize';
2902
2903     if ($conf->{smbios1}) {
2904         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2905     }
2906
2907     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2908         die "uefi base image not found\n" if ! -f $OVMF_CODE;
2909
2910         my $path;
2911         my $format = 'raw';
2912         if (my $efidisk = $conf->{efidisk0}) {
2913             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
2914             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2915             if ($storeid) {
2916                 $path = PVE::Storage::path($storecfg, $d->{file});
2917             } else {
2918                 $path = $d->{file};
2919             }
2920             $format = $d->{format} if $d->{format};
2921         } else {
2922             warn "no efidisk configured! Using temporary efivars disk.\n";
2923             $path = "/tmp/$vmid-ovmf.fd";
2924             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
2925         }
2926
2927         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
2928         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
2929     }
2930
2931
2932     # add usb controllers
2933     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2934     push @$devices, @usbcontrollers if @usbcontrollers;
2935     my $vga = $conf->{vga};
2936
2937     my $qxlnum = vga_conf_has_spice($vga);
2938     $vga = 'qxl' if $qxlnum;
2939
2940     if (!$vga) {
2941         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
2942             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
2943         } else {
2944             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
2945         }
2946     }
2947
2948     # enable absolute mouse coordinates (needed by vnc)
2949     my $tablet;
2950     if (defined($conf->{tablet})) {
2951         $tablet = $conf->{tablet};
2952     } else {
2953         $tablet = $defaults->{tablet};
2954         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2955         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2956     }
2957
2958     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2959
2960     my $kvm_off = 0;
2961     my $gpu_passthrough;
2962
2963     # host pci devices
2964     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2965         my $d = parse_hostpci($conf->{"hostpci$i"});
2966         next if !$d;
2967
2968         my $pcie = $d->{pcie};
2969         if($pcie){
2970             die "q35 machine model is not enabled" if !$q35;
2971             $pciaddr = print_pcie_addr("hostpci$i");
2972         }else{
2973             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2974         }
2975
2976         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2977         my $romfile = $d->{romfile};
2978
2979         my $xvga = '';
2980         if ($d->{'x-vga'}) {
2981             $xvga = ',x-vga=on';
2982             $kvm_off = 1;
2983             $vga = 'none';
2984             $gpu_passthrough = 1;
2985
2986             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2987                 $xvga = "";
2988             }
2989         }
2990         my $pcidevices = $d->{pciid};
2991         my $multifunction = 1 if @$pcidevices > 1;
2992
2993         my $j=0;
2994         foreach my $pcidevice (@$pcidevices) {
2995
2996             my $id = "hostpci$i";
2997             $id .= ".$j" if $multifunction;
2998             my $addr = $pciaddr;
2999             $addr .= ".$j" if $multifunction;
3000             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3001
3002             if($j == 0){
3003                 $devicestr .= "$rombar$xvga";
3004                 $devicestr .= ",multifunction=on" if $multifunction;
3005                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3006             }
3007
3008             push @$devices, '-device', $devicestr;
3009             $j++;
3010         }
3011     }
3012
3013     # usb devices
3014     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3015     push @$devices, @usbdevices if @usbdevices;
3016     # serial devices
3017     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3018         if (my $path = $conf->{"serial$i"}) {
3019             if ($path eq 'socket') {
3020                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3021                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3022                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3023             } else {
3024                 die "no such serial device\n" if ! -c $path;
3025                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3026                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3027             }
3028         }
3029     }
3030
3031     # parallel devices
3032     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3033         if (my $path = $conf->{"parallel$i"}) {
3034             die "no such parallel device\n" if ! -c $path;
3035             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3036             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3037             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3038         }
3039     }
3040
3041     my $vmname = $conf->{name} || "vm$vmid";
3042
3043     push @$cmd, '-name', $vmname;
3044
3045     my $sockets = 1;
3046     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3047     $sockets = $conf->{sockets} if  $conf->{sockets};
3048
3049     my $cores = $conf->{cores} || 1;
3050
3051     my $maxcpus = $sockets * $cores;
3052
3053     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3054
3055     my $allowed_vcpus = $cpuinfo->{cpus};
3056
3057     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3058         if ($allowed_vcpus < $maxcpus);
3059
3060     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3061
3062         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3063         for (my $i = 2; $i <= $vcpus; $i++)  {
3064             my $cpustr = print_cpu_device($conf,$i);
3065             push @$cmd, '-device', $cpustr;
3066         }
3067
3068     } else {
3069
3070         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3071     }
3072     push @$cmd, '-nodefaults';
3073
3074     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3075
3076     my $bootindex_hash = {};
3077     my $i = 1;
3078     foreach my $o (split(//, $bootorder)) {
3079         $bootindex_hash->{$o} = $i*100;
3080         $i++;
3081     }
3082
3083     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3084
3085     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3086
3087     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3088
3089     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3090
3091     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3092         my $socket = vnc_socket($vmid);
3093         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3094     } else {
3095         push @$cmd, '-nographic';
3096     }
3097
3098     # time drift fix
3099     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3100
3101     my $useLocaltime = $conf->{localtime};
3102
3103     if ($winversion >= 5) { # windows
3104         $useLocaltime = 1 if !defined($conf->{localtime});
3105
3106         # use time drift fix when acpi is enabled
3107         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3108             $tdf = 1 if !defined($conf->{tdf});
3109         }
3110     }
3111
3112     if ($winversion >= 6) {
3113         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3114         push @$cmd, '-no-hpet';
3115     }
3116
3117     push @$rtcFlags, 'driftfix=slew' if $tdf;
3118
3119     if (!$kvm) {
3120         push @$machineFlags, 'accel=tcg';
3121     }
3122
3123     if ($machine_type) {
3124         push @$machineFlags, "type=${machine_type}";
3125     }
3126
3127     if ($conf->{startdate}) {
3128         push @$rtcFlags, "base=$conf->{startdate}";
3129     } elsif ($useLocaltime) {
3130         push @$rtcFlags, 'base=localtime';
3131     }
3132
3133     my $cpu = $kvm ? "kvm64" : "qemu64";
3134     if (my $cputype = $conf->{cpu}) {
3135         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3136             or die "Cannot parse cpu description: $cputype\n";
3137         $cpu = $cpuconf->{cputype};
3138         $kvm_off = 1 if $cpuconf->{hidden};
3139     }
3140
3141     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3142
3143     push @$cpuFlags , '-x2apic'
3144         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3145
3146     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3147
3148     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3149
3150     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3151
3152         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3153         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3154     }
3155
3156     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3157
3158     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3159
3160     push @$cpuFlags, 'kvm=off' if $kvm_off;
3161
3162     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3163         die "internal error"; # should not happen
3164
3165     push @$cpuFlags, "vendor=${cpu_vendor}"
3166         if $cpu_vendor ne 'default';
3167
3168     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3169
3170     push @$cmd, '-cpu', $cpu;
3171
3172     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3173
3174     push @$cmd, '-S' if $conf->{freeze};
3175
3176     # set keyboard layout
3177     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3178     push @$cmd, '-k', $kb if $kb;
3179
3180     # enable sound
3181     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3182     #push @$cmd, '-soundhw', 'es1370';
3183     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3184
3185     if($conf->{agent}) {
3186         my $qgasocket = qmp_socket($vmid, 1);
3187         my $pciaddr = print_pci_addr("qga0", $bridges);
3188         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3189         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3190         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3191     }
3192
3193     my $spice_port;
3194
3195     if ($qxlnum) {
3196         if ($qxlnum > 1) {
3197             if ($winversion){
3198                 for(my $i = 1; $i < $qxlnum; $i++){
3199                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3200                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3201                 }
3202             } else {
3203                 # assume other OS works like Linux
3204                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3205                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3206             }
3207         }
3208
3209         my $pciaddr = print_pci_addr("spice", $bridges);
3210
3211         my $nodename = PVE::INotify::nodename();
3212         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3213         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3214         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3215         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3216         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3217
3218         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3219
3220         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3221         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3222         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3223     }
3224
3225     # enable balloon by default, unless explicitly disabled
3226     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3227         $pciaddr = print_pci_addr("balloon0", $bridges);
3228         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3229     }
3230
3231     if ($conf->{watchdog}) {
3232         my $wdopts = parse_watchdog($conf->{watchdog});
3233         $pciaddr = print_pci_addr("watchdog", $bridges);
3234         my $watchdog = $wdopts->{model} || 'i6300esb';
3235         push @$devices, '-device', "$watchdog$pciaddr";
3236         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3237     }
3238
3239     my $vollist = [];
3240     my $scsicontroller = {};
3241     my $ahcicontroller = {};
3242     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3243
3244     # Add iscsi initiator name if available
3245     if (my $initiator = get_initiator_name()) {
3246         push @$devices, '-iscsi', "initiator-name=$initiator";
3247     }
3248
3249     foreach_drive($conf, sub {
3250         my ($ds, $drive) = @_;
3251
3252         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3253             push @$vollist, $drive->{file};
3254         }
3255
3256         # ignore efidisk here, already added in bios/fw handling code above
3257         return if $drive->{interface} eq 'efidisk';
3258
3259         $use_virtio = 1 if $ds =~ m/^virtio/;
3260
3261         if (drive_is_cdrom ($drive)) {
3262             if ($bootindex_hash->{d}) {
3263                 $drive->{bootindex} = $bootindex_hash->{d};
3264                 $bootindex_hash->{d} += 1;
3265             }
3266         } else {
3267             if ($bootindex_hash->{c}) {
3268                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3269                 $bootindex_hash->{c} += 1;
3270             }
3271         }
3272
3273         if($drive->{interface} eq 'virtio'){
3274            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3275         }
3276
3277         if ($drive->{interface} eq 'scsi') {
3278
3279             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3280
3281             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3282             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3283
3284             my $iothread = '';
3285             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3286                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3287                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3288             } elsif ($drive->{iothread}) {
3289                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3290             }
3291
3292             my $queues = '';
3293             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3294                 $queues = ",num_queues=$drive->{queues}";
3295             }
3296
3297             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3298             $scsicontroller->{$controller}=1;
3299         }
3300
3301         if ($drive->{interface} eq 'sata') {
3302            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3303            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3304            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3305            $ahcicontroller->{$controller}=1;
3306         }
3307
3308         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3309         push @$devices, '-drive',$drive_cmd;
3310         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3311     });
3312
3313     for (my $i = 0; $i < $MAX_NETS; $i++) {
3314          next if !$conf->{"net$i"};
3315          my $d = parse_net($conf->{"net$i"});
3316          next if !$d;
3317
3318          $use_virtio = 1 if $d->{model} eq 'virtio';
3319
3320          if ($bootindex_hash->{n}) {
3321             $d->{bootindex} = $bootindex_hash->{n};
3322             $bootindex_hash->{n} += 1;
3323          }
3324
3325          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3326          push @$devices, '-netdev', $netdevfull;
3327
3328          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3329          push @$devices, '-device', $netdevicefull;
3330     }
3331
3332     if (!$q35) {
3333         # add pci bridges
3334         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3335            $bridges->{1} = 1;
3336            $bridges->{2} = 1;
3337         }
3338
3339         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3340
3341         while (my ($k, $v) = each %$bridges) {
3342             $pciaddr = print_pci_addr("pci.$k");
3343             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3344         }
3345     }
3346
3347     # add custom args
3348     if ($conf->{args}) {
3349         my $aa = PVE::Tools::split_args($conf->{args});
3350         push @$cmd, @$aa;
3351     }
3352
3353     push @$cmd, @$devices;
3354     push @$cmd, '-rtc', join(',', @$rtcFlags)
3355         if scalar(@$rtcFlags);
3356     push @$cmd, '-machine', join(',', @$machineFlags)
3357         if scalar(@$machineFlags);
3358     push @$cmd, '-global', join(',', @$globalFlags)
3359         if scalar(@$globalFlags);
3360
3361     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3362 }
3363
3364 sub vnc_socket {
3365     my ($vmid) = @_;
3366     return "${var_run_tmpdir}/$vmid.vnc";
3367 }
3368
3369 sub spice_port {
3370     my ($vmid) = @_;
3371
3372     my $res = vm_mon_cmd($vmid, 'query-spice');
3373
3374     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3375 }
3376
3377 sub qmp_socket {
3378     my ($vmid, $qga) = @_;
3379     my $sockettype = $qga ? 'qga' : 'qmp';
3380     return "${var_run_tmpdir}/$vmid.$sockettype";
3381 }
3382
3383 sub pidfile_name {
3384     my ($vmid) = @_;
3385     return "${var_run_tmpdir}/$vmid.pid";
3386 }
3387
3388 sub vm_devices_list {
3389     my ($vmid) = @_;
3390
3391     my $res = vm_mon_cmd($vmid, 'query-pci');
3392     my $devices = {};
3393     foreach my $pcibus (@$res) {
3394         foreach my $device (@{$pcibus->{devices}}) {
3395             next if !$device->{'qdev_id'};
3396             if ($device->{'pci_bridge'}) {
3397                 $devices->{$device->{'qdev_id'}} = 1;
3398                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3399                     next if !$bridge_device->{'qdev_id'};
3400                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3401                     $devices->{$device->{'qdev_id'}}++;
3402                 }
3403             } else {
3404                 $devices->{$device->{'qdev_id'}} = 1;
3405             }
3406         }
3407     }
3408
3409     my $resblock = vm_mon_cmd($vmid, 'query-block');
3410     foreach my $block (@$resblock) {
3411         if($block->{device} =~ m/^drive-(\S+)/){
3412                 $devices->{$1} = 1;
3413         }
3414     }
3415
3416     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3417     foreach my $mice (@$resmice) {
3418         if ($mice->{name} eq 'QEMU HID Tablet') {
3419             $devices->{tablet} = 1;
3420             last;
3421         }
3422     }
3423
3424     # for usb devices there is no query-usb
3425     # but we can iterate over the entries in
3426     # qom-list path=/machine/peripheral
3427     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3428     foreach my $per (@$resperipheral) {
3429         if ($per->{name} =~ m/^usb\d+$/) {
3430             $devices->{$per->{name}} = 1;
3431         }
3432     }
3433
3434     return $devices;
3435 }
3436
3437 sub vm_deviceplug {
3438     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3439
3440     my $q35 = machine_type_is_q35($conf);
3441
3442     my $devices_list = vm_devices_list($vmid);
3443     return 1 if defined($devices_list->{$deviceid});
3444
3445     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3446
3447     if ($deviceid eq 'tablet') {
3448
3449         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3450
3451     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3452
3453         die "usb hotplug currently not reliable\n";
3454         # since we can't reliably hot unplug all added usb devices
3455         # and usb passthrough disables live migration
3456         # we disable usb hotplugging for now
3457         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3458
3459     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3460
3461         qemu_iothread_add($vmid, $deviceid, $device);
3462
3463         qemu_driveadd($storecfg, $vmid, $device);
3464         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3465
3466         qemu_deviceadd($vmid, $devicefull);
3467         eval { qemu_deviceaddverify($vmid, $deviceid); };
3468         if (my $err = $@) {
3469             eval { qemu_drivedel($vmid, $deviceid); };
3470             warn $@ if $@;
3471             die $err;
3472         }
3473
3474     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3475
3476
3477         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3478         my $pciaddr = print_pci_addr($deviceid);
3479         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3480
3481         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3482
3483         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3484             qemu_iothread_add($vmid, $deviceid, $device);
3485             $devicefull .= ",iothread=iothread-$deviceid";
3486         }
3487
3488         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3489             $devicefull .= ",num_queues=$device->{queues}";
3490         }
3491
3492         qemu_deviceadd($vmid, $devicefull);
3493         qemu_deviceaddverify($vmid, $deviceid);
3494
3495     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3496
3497         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3498         qemu_driveadd($storecfg, $vmid, $device);
3499
3500         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3501         eval { qemu_deviceadd($vmid, $devicefull); };
3502         if (my $err = $@) {
3503             eval { qemu_drivedel($vmid, $deviceid); };
3504             warn $@ if $@;
3505             die $err;
3506         }
3507
3508     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3509
3510         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3511
3512         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3513         my $use_old_bios_files = undef;
3514         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3515
3516         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3517         qemu_deviceadd($vmid, $netdevicefull);
3518         eval { qemu_deviceaddverify($vmid, $deviceid); };
3519         if (my $err = $@) {
3520             eval { qemu_netdevdel($vmid, $deviceid); };
3521             warn $@ if $@;
3522             die $err;
3523         }
3524
3525     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3526
3527         my $bridgeid = $2;
3528         my $pciaddr = print_pci_addr($deviceid);
3529         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3530
3531         qemu_deviceadd($vmid, $devicefull);
3532         qemu_deviceaddverify($vmid, $deviceid);
3533
3534     } else {
3535         die "can't hotplug device '$deviceid'\n";
3536     }
3537
3538     return 1;
3539 }
3540
3541 # fixme: this should raise exceptions on error!
3542 sub vm_deviceunplug {
3543     my ($vmid, $conf, $deviceid) = @_;
3544
3545     my $devices_list = vm_devices_list($vmid);
3546     return 1 if !defined($devices_list->{$deviceid});
3547
3548     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3549
3550     if ($deviceid eq 'tablet') {
3551
3552         qemu_devicedel($vmid, $deviceid);
3553
3554     } elsif ($deviceid =~ m/^usb\d+$/) {
3555
3556         die "usb hotplug currently not reliable\n";
3557         # when unplugging usb devices this way,
3558         # there may be remaining usb controllers/hubs
3559         # so we disable it for now
3560         qemu_devicedel($vmid, $deviceid);
3561         qemu_devicedelverify($vmid, $deviceid);
3562
3563     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3564
3565         qemu_devicedel($vmid, $deviceid);
3566         qemu_devicedelverify($vmid, $deviceid);
3567         qemu_drivedel($vmid, $deviceid);
3568         qemu_iothread_del($conf, $vmid, $deviceid);
3569
3570     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3571
3572         qemu_devicedel($vmid, $deviceid);
3573         qemu_devicedelverify($vmid, $deviceid);
3574         qemu_iothread_del($conf, $vmid, $deviceid);
3575
3576     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3577
3578         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3579         my $device = parse_drive($deviceid, $conf->{$deviceid});
3580         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3581
3582         qemu_devicedel($vmid, $deviceid);
3583         qemu_drivedel($vmid, $deviceid);
3584         qemu_deletescsihw($conf, $vmid, $deviceid);
3585
3586     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3587
3588         qemu_devicedel($vmid, $deviceid);
3589         qemu_devicedelverify($vmid, $deviceid);
3590         qemu_netdevdel($vmid, $deviceid);
3591
3592     } else {
3593         die "can't unplug device '$deviceid'\n";
3594     }
3595
3596     return 1;
3597 }
3598
3599 sub qemu_deviceadd {
3600     my ($vmid, $devicefull) = @_;
3601
3602     $devicefull = "driver=".$devicefull;
3603     my %options =  split(/[=,]/, $devicefull);
3604
3605     vm_mon_cmd($vmid, "device_add" , %options);
3606 }
3607
3608 sub qemu_devicedel {
3609     my ($vmid, $deviceid) = @_;
3610
3611     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3612 }
3613
3614 sub qemu_iothread_add {
3615     my($vmid, $deviceid, $device) = @_;
3616
3617     if ($device->{iothread}) {
3618         my $iothreads = vm_iothreads_list($vmid);
3619         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3620     }
3621 }
3622
3623 sub qemu_iothread_del {
3624     my($conf, $vmid, $deviceid) = @_;
3625
3626     my $device = parse_drive($deviceid, $conf->{$deviceid});
3627     if ($device->{iothread}) {
3628         my $iothreads = vm_iothreads_list($vmid);
3629         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3630     }
3631 }
3632
3633 sub qemu_objectadd {
3634     my($vmid, $objectid, $qomtype) = @_;
3635
3636     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3637
3638     return 1;
3639 }
3640
3641 sub qemu_objectdel {
3642     my($vmid, $objectid) = @_;
3643
3644     vm_mon_cmd($vmid, "object-del", id => $objectid);
3645
3646     return 1;
3647 }
3648
3649 sub qemu_driveadd {
3650     my ($storecfg, $vmid, $device) = @_;
3651
3652     my $drive = print_drive_full($storecfg, $vmid, $device);
3653     $drive =~ s/\\/\\\\/g;
3654     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3655
3656     # If the command succeeds qemu prints: "OK"
3657     return 1 if $ret =~ m/OK/s;
3658
3659     die "adding drive failed: $ret\n";
3660 }
3661
3662 sub qemu_drivedel {
3663     my($vmid, $deviceid) = @_;
3664
3665     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3666     $ret =~ s/^\s+//;
3667
3668     return 1 if $ret eq "";
3669
3670     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3671     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3672
3673     die "deleting drive $deviceid failed : $ret\n";
3674 }
3675
3676 sub qemu_deviceaddverify {
3677     my ($vmid, $deviceid) = @_;
3678
3679     for (my $i = 0; $i <= 5; $i++) {
3680          my $devices_list = vm_devices_list($vmid);
3681          return 1 if defined($devices_list->{$deviceid});
3682          sleep 1;
3683     }
3684
3685     die "error on hotplug device '$deviceid'\n";
3686 }
3687
3688
3689 sub qemu_devicedelverify {
3690     my ($vmid, $deviceid) = @_;
3691
3692     # need to verify that the device is correctly removed as device_del
3693     # is async and empty return is not reliable
3694
3695     for (my $i = 0; $i <= 5; $i++) {
3696          my $devices_list = vm_devices_list($vmid);
3697          return 1 if !defined($devices_list->{$deviceid});
3698          sleep 1;
3699     }
3700
3701     die "error on hot-unplugging device '$deviceid'\n";
3702 }
3703
3704 sub qemu_findorcreatescsihw {
3705     my ($storecfg, $conf, $vmid, $device) = @_;
3706
3707     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3708
3709     my $scsihwid="$controller_prefix$controller";
3710     my $devices_list = vm_devices_list($vmid);
3711
3712     if(!defined($devices_list->{$scsihwid})) {
3713         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3714     }
3715
3716     return 1;
3717 }
3718
3719 sub qemu_deletescsihw {
3720     my ($conf, $vmid, $opt) = @_;
3721
3722     my $device = parse_drive($opt, $conf->{$opt});
3723
3724     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3725         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3726         return 1;
3727     }
3728
3729     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3730
3731     my $devices_list = vm_devices_list($vmid);
3732     foreach my $opt (keys %{$devices_list}) {
3733         if (PVE::QemuServer::is_valid_drivename($opt)) {
3734             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3735             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3736                 return 1;
3737             }
3738         }
3739     }
3740
3741     my $scsihwid="scsihw$controller";
3742
3743     vm_deviceunplug($vmid, $conf, $scsihwid);
3744
3745     return 1;
3746 }
3747
3748 sub qemu_add_pci_bridge {
3749     my ($storecfg, $conf, $vmid, $device) = @_;
3750
3751     my $bridges = {};
3752
3753     my $bridgeid;
3754
3755     print_pci_addr($device, $bridges);
3756
3757     while (my ($k, $v) = each %$bridges) {
3758         $bridgeid = $k;
3759     }
3760     return 1 if !defined($bridgeid) || $bridgeid < 1;
3761
3762     my $bridge = "pci.$bridgeid";
3763     my $devices_list = vm_devices_list($vmid);
3764
3765     if (!defined($devices_list->{$bridge})) {
3766         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3767     }
3768
3769     return 1;
3770 }
3771
3772 sub qemu_set_link_status {
3773     my ($vmid, $device, $up) = @_;
3774
3775     vm_mon_cmd($vmid, "set_link", name => $device,
3776                up => $up ? JSON::true : JSON::false);
3777 }
3778
3779 sub qemu_netdevadd {
3780     my ($vmid, $conf, $device, $deviceid) = @_;
3781
3782     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3783     my %options =  split(/[=,]/, $netdev);
3784
3785     vm_mon_cmd($vmid, "netdev_add",  %options);
3786     return 1;
3787 }
3788
3789 sub qemu_netdevdel {
3790     my ($vmid, $deviceid) = @_;
3791
3792     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3793 }
3794
3795 sub qemu_usb_hotplug {
3796     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3797
3798     return if !$device;
3799
3800     # remove the old one first
3801     vm_deviceunplug($vmid, $conf, $deviceid);
3802
3803     # check if xhci controller is necessary and available
3804     if ($device->{usb3}) {
3805
3806         my $devicelist = vm_devices_list($vmid);
3807
3808         if (!$devicelist->{xhci}) {
3809             my $pciaddr = print_pci_addr("xhci");
3810             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3811         }
3812     }
3813     my $d = parse_usb_device($device->{host});
3814     $d->{usb3} = $device->{usb3};
3815
3816     # add the new one
3817     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3818 }
3819
3820 sub qemu_cpu_hotplug {
3821     my ($vmid, $conf, $vcpus) = @_;
3822
3823     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3824
3825     my $sockets = 1;
3826     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3827     $sockets = $conf->{sockets} if  $conf->{sockets};
3828     my $cores = $conf->{cores} || 1;
3829     my $maxcpus = $sockets * $cores;
3830
3831     $vcpus = $maxcpus if !$vcpus;
3832
3833     die "you can't add more vcpus than maxcpus\n"
3834         if $vcpus > $maxcpus;
3835
3836     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3837
3838     if ($vcpus < $currentvcpus) {
3839
3840         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3841
3842             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3843                 qemu_devicedel($vmid, "cpu$i");
3844                 my $retry = 0;
3845                 my $currentrunningvcpus = undef;
3846                 while (1) {
3847                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3848                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3849                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3850                     $retry++;
3851                     sleep 1;
3852                 }
3853                 #update conf after each succesfull cpu unplug
3854                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3855                 PVE::QemuConfig->write_config($vmid, $conf);
3856             }
3857         } else {
3858             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3859         }
3860
3861         return;
3862     }
3863
3864     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3865     die "vcpus in running vm does not match its configuration\n"
3866         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3867
3868     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3869
3870         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3871             my $cpustr = print_cpu_device($conf, $i);
3872             qemu_deviceadd($vmid, $cpustr);
3873
3874             my $retry = 0;
3875             my $currentrunningvcpus = undef;
3876             while (1) {
3877                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3878                 last if scalar(@{$currentrunningvcpus}) == $i;
3879                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3880                 sleep 1;
3881                 $retry++;
3882             }
3883             #update conf after each succesfull cpu hotplug
3884             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3885             PVE::QemuConfig->write_config($vmid, $conf);
3886         }
3887     } else {
3888
3889         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3890             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3891         }
3892     }
3893 }
3894
3895 sub qemu_block_set_io_throttle {
3896     my ($vmid, $deviceid,
3897         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3898         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3899         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3900         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3901
3902     return if !check_running($vmid) ;
3903
3904     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3905         bps => int($bps),
3906         bps_rd => int($bps_rd),
3907         bps_wr => int($bps_wr),
3908         iops => int($iops),
3909         iops_rd => int($iops_rd),
3910         iops_wr => int($iops_wr),
3911         bps_max => int($bps_max),
3912         bps_rd_max => int($bps_rd_max),
3913         bps_wr_max => int($bps_wr_max),
3914         iops_max => int($iops_max),
3915         iops_rd_max => int($iops_rd_max),
3916         iops_wr_max => int($iops_wr_max),
3917         bps_max_length => int($bps_max_length),
3918         bps_rd_max_length => int($bps_rd_max_length),
3919         bps_wr_max_length => int($bps_wr_max_length),
3920         iops_max_length => int($iops_max_length),
3921         iops_rd_max_length => int($iops_rd_max_length),
3922         iops_wr_max_length => int($iops_wr_max_length),
3923     );
3924
3925 }
3926
3927 # old code, only used to shutdown old VM after update
3928 sub __read_avail {
3929     my ($fh, $timeout) = @_;
3930
3931     my $sel = new IO::Select;
3932     $sel->add($fh);
3933
3934     my $res = '';
3935     my $buf;
3936
3937     my @ready;
3938     while (scalar (@ready = $sel->can_read($timeout))) {
3939         my $count;
3940         if ($count = $fh->sysread($buf, 8192)) {
3941             if ($buf =~ /^(.*)\(qemu\) $/s) {
3942                 $res .= $1;
3943                 last;
3944             } else {
3945                 $res .= $buf;
3946             }
3947         } else {
3948             if (!defined($count)) {
3949                 die "$!\n";
3950             }
3951             last;
3952         }
3953     }
3954
3955     die "monitor read timeout\n" if !scalar(@ready);
3956
3957     return $res;
3958 }
3959
3960 # old code, only used to shutdown old VM after update
3961 sub vm_monitor_command {
3962     my ($vmid, $cmdstr, $nocheck) = @_;
3963
3964     my $res;
3965
3966     eval {
3967         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3968
3969         my $sname = "${var_run_tmpdir}/$vmid.mon";
3970
3971         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3972             die "unable to connect to VM $vmid socket - $!\n";
3973
3974         my $timeout = 3;
3975
3976         # hack: migrate sometime blocks the monitor (when migrate_downtime
3977         # is set)
3978         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3979             $timeout = 60*60; # 1 hour
3980         }
3981
3982         # read banner;
3983         my $data = __read_avail($sock, $timeout);
3984
3985         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3986             die "got unexpected qemu monitor banner\n";
3987         }
3988
3989         my $sel = new IO::Select;
3990         $sel->add($sock);
3991
3992         if (!scalar(my @ready = $sel->can_write($timeout))) {
3993             die "monitor write error - timeout";
3994         }
3995
3996         my $fullcmd = "$cmdstr\r";
3997
3998         # syslog('info', "VM $vmid monitor command: $cmdstr");
3999
4000         my $b;
4001         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
4002             die "monitor write error - $!";
4003         }
4004
4005         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4006
4007         $timeout = 20;
4008
4009         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4010             $timeout = 60*60; # 1 hour
4011         } elsif ($cmdstr =~ m/^(eject|change)/) {
4012             $timeout = 60; # note: cdrom mount command is slow
4013         }
4014         if ($res = __read_avail($sock, $timeout)) {
4015
4016             my @lines = split("\r?\n", $res);
4017
4018             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4019
4020             $res = join("\n", @lines);
4021             $res .= "\n";
4022         }
4023     };
4024
4025     my $err = $@;
4026
4027     if ($err) {
4028         syslog("err", "VM $vmid monitor command failed - $err");
4029         die $err;
4030     }
4031
4032     return $res;
4033 }
4034
4035 sub qemu_block_resize {
4036     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4037
4038     my $running = check_running($vmid);
4039
4040     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4041
4042     return if !$running;
4043
4044     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4045
4046 }
4047
4048 sub qemu_volume_snapshot {
4049     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4050
4051     my $running = check_running($vmid);
4052
4053     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4054         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4055     } else {
4056         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4057     }
4058 }
4059
4060 sub qemu_volume_snapshot_delete {
4061     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4062
4063     my $running = check_running($vmid);
4064
4065     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4066         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4067     } else {
4068         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4069     }
4070 }
4071
4072 sub set_migration_caps {
4073     my ($vmid) = @_;
4074
4075     my $cap_ref = [];
4076
4077     my $enabled_cap = {
4078         "auto-converge" => 1,
4079         "xbzrle" => 1,
4080         "x-rdma-pin-all" => 0,
4081         "zero-blocks" => 0,
4082         "compress" => 0
4083     };
4084
4085     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4086
4087     for my $supported_capability (@$supported_capabilities) {
4088         push @$cap_ref, {
4089             capability => $supported_capability->{capability},
4090             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4091         };
4092     }
4093
4094     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4095 }
4096
4097 my $fast_plug_option = {
4098     'lock' => 1,
4099     'name' => 1,
4100     'onboot' => 1,
4101     'shares' => 1,
4102     'startup' => 1,
4103     'description' => 1,
4104     'protection' => 1,
4105     'vmstatestorage' => 1,
4106 };
4107
4108 # hotplug changes in [PENDING]
4109 # $selection hash can be used to only apply specified options, for
4110 # example: { cores => 1 } (only apply changed 'cores')
4111 # $errors ref is used to return error messages
4112 sub vmconfig_hotplug_pending {
4113     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4114
4115     my $defaults = load_defaults();
4116
4117     # commit values which do not have any impact on running VM first
4118     # Note: those option cannot raise errors, we we do not care about
4119     # $selection and always apply them.
4120
4121     my $add_error = sub {
4122         my ($opt, $msg) = @_;
4123         $errors->{$opt} = "hotplug problem - $msg";
4124     };
4125
4126     my $changes = 0;
4127     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4128         if ($fast_plug_option->{$opt}) {
4129             $conf->{$opt} = $conf->{pending}->{$opt};
4130             delete $conf->{pending}->{$opt};
4131             $changes = 1;
4132         }
4133     }
4134
4135     if ($changes) {
4136         PVE::QemuConfig->write_config($vmid, $conf);
4137         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4138     }
4139
4140     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4141
4142     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4143     while (my ($opt, $force) = each %$pending_delete_hash) {
4144         next if $selection && !$selection->{$opt};
4145         eval {
4146             if ($opt eq 'hotplug') {
4147                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4148             } elsif ($opt eq 'tablet') {
4149                 die "skip\n" if !$hotplug_features->{usb};
4150                 if ($defaults->{tablet}) {
4151                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4152                 } else {
4153                     vm_deviceunplug($vmid, $conf, $opt);
4154                 }
4155             } elsif ($opt =~ m/^usb\d+/) {
4156                 die "skip\n";
4157                 # since we cannot reliably hot unplug usb devices
4158                 # we are disabling it
4159                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4160                 vm_deviceunplug($vmid, $conf, $opt);
4161             } elsif ($opt eq 'vcpus') {
4162                 die "skip\n" if !$hotplug_features->{cpu};
4163                 qemu_cpu_hotplug($vmid, $conf, undef);
4164             } elsif ($opt eq 'balloon') {
4165                 # enable balloon device is not hotpluggable
4166                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4167             } elsif ($fast_plug_option->{$opt}) {
4168                 # do nothing
4169             } elsif ($opt =~ m/^net(\d+)$/) {
4170                 die "skip\n" if !$hotplug_features->{network};
4171                 vm_deviceunplug($vmid, $conf, $opt);
4172             } elsif (is_valid_drivename($opt)) {
4173                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4174                 vm_deviceunplug($vmid, $conf, $opt);
4175                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4176             } elsif ($opt =~ m/^memory$/) {
4177                 die "skip\n" if !$hotplug_features->{memory};
4178                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4179             } elsif ($opt eq 'cpuunits') {
4180                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4181             } elsif ($opt eq 'cpulimit') {
4182                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4183             } else {
4184                 die "skip\n";
4185             }
4186         };
4187         if (my $err = $@) {
4188             &$add_error($opt, $err) if $err ne "skip\n";
4189         } else {
4190             # save new config if hotplug was successful
4191             delete $conf->{$opt};
4192             vmconfig_undelete_pending_option($conf, $opt);
4193             PVE::QemuConfig->write_config($vmid, $conf);
4194             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4195         }
4196     }
4197
4198     foreach my $opt (keys %{$conf->{pending}}) {
4199         next if $selection && !$selection->{$opt};
4200         my $value = $conf->{pending}->{$opt};
4201         eval {
4202             if ($opt eq 'hotplug') {
4203                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4204             } elsif ($opt eq 'tablet') {
4205                 die "skip\n" if !$hotplug_features->{usb};
4206                 if ($value == 1) {
4207                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4208                 } elsif ($value == 0) {
4209                     vm_deviceunplug($vmid, $conf, $opt);
4210                 }
4211             } elsif ($opt =~ m/^usb\d+$/) {
4212                 die "skip\n";
4213                 # since we cannot reliably hot unplug usb devices
4214                 # we are disabling it
4215                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4216                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4217                 die "skip\n" if !$d;
4218                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4219             } elsif ($opt eq 'vcpus') {
4220                 die "skip\n" if !$hotplug_features->{cpu};
4221                 qemu_cpu_hotplug($vmid, $conf, $value);
4222             } elsif ($opt eq 'balloon') {
4223                 # enable/disable balloning device is not hotpluggable
4224                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4225                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4226                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4227
4228                 # allow manual ballooning if shares is set to zero
4229                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4230                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4231                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4232                 }
4233             } elsif ($opt =~ m/^net(\d+)$/) {
4234                 # some changes can be done without hotplug
4235                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4236                                     $vmid, $opt, $value);
4237             } elsif (is_valid_drivename($opt)) {
4238                 # some changes can be done without hotplug
4239                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4240                                      $vmid, $opt, $value, 1);
4241             } elsif ($opt =~ m/^memory$/) { #dimms
4242                 die "skip\n" if !$hotplug_features->{memory};
4243                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4244             } elsif ($opt eq 'cpuunits') {
4245                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4246             } elsif ($opt eq 'cpulimit') {
4247                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4248                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4249             } else {
4250                 die "skip\n";  # skip non-hot-pluggable options
4251             }
4252         };
4253         if (my $err = $@) {
4254             &$add_error($opt, $err) if $err ne "skip\n";
4255         } else {
4256             # save new config if hotplug was successful
4257             $conf->{$opt} = $value;
4258             delete $conf->{pending}->{$opt};
4259             PVE::QemuConfig->write_config($vmid, $conf);
4260             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4261         }
4262     }
4263 }
4264
4265 sub try_deallocate_drive {
4266     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4267
4268     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4269         my $volid = $drive->{file};
4270         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4271             my $sid = PVE::Storage::parse_volume_id($volid);
4272             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4273
4274             # check if the disk is really unused
4275             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4276                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4277             PVE::Storage::vdisk_free($storecfg, $volid);
4278             return 1;
4279         } else {
4280             # If vm is not owner of this disk remove from config
4281             return 1;
4282         }
4283     }
4284
4285     return undef;
4286 }
4287
4288 sub vmconfig_delete_or_detach_drive {
4289     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4290
4291     my $drive = parse_drive($opt, $conf->{$opt});
4292
4293     my $rpcenv = PVE::RPCEnvironment::get();
4294     my $authuser = $rpcenv->get_user();
4295
4296     if ($force) {
4297         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4298         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4299     } else {
4300         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4301     }
4302 }
4303
4304 sub vmconfig_apply_pending {
4305     my ($vmid, $conf, $storecfg) = @_;
4306
4307     # cold plug
4308
4309     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4310     while (my ($opt, $force) = each %$pending_delete_hash) {
4311         die "internal error" if $opt =~ m/^unused/;
4312         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4313         if (!defined($conf->{$opt})) {
4314             vmconfig_undelete_pending_option($conf, $opt);
4315             PVE::QemuConfig->write_config($vmid, $conf);
4316         } elsif (is_valid_drivename($opt)) {
4317             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4318             vmconfig_undelete_pending_option($conf, $opt);
4319             delete $conf->{$opt};
4320             PVE::QemuConfig->write_config($vmid, $conf);
4321         } else {
4322             vmconfig_undelete_pending_option($conf, $opt);
4323             delete $conf->{$opt};
4324             PVE::QemuConfig->write_config($vmid, $conf);
4325         }
4326     }
4327
4328     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4329
4330     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4331         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4332
4333         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4334             # skip if nothing changed
4335         } elsif (is_valid_drivename($opt)) {
4336             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4337                 if defined($conf->{$opt});
4338             $conf->{$opt} = $conf->{pending}->{$opt};
4339         } else {
4340             $conf->{$opt} = $conf->{pending}->{$opt};
4341         }
4342
4343         delete $conf->{pending}->{$opt};
4344         PVE::QemuConfig->write_config($vmid, $conf);
4345     }
4346 }
4347
4348 my $safe_num_ne = sub {
4349     my ($a, $b) = @_;
4350
4351     return 0 if !defined($a) && !defined($b);
4352     return 1 if !defined($a);
4353     return 1 if !defined($b);
4354
4355     return $a != $b;
4356 };
4357
4358 my $safe_string_ne = sub {
4359     my ($a, $b) = @_;
4360
4361     return 0 if !defined($a) && !defined($b);
4362     return 1 if !defined($a);
4363     return 1 if !defined($b);
4364
4365     return $a ne $b;
4366 };
4367
4368 sub vmconfig_update_net {
4369     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4370
4371     my $newnet = parse_net($value);
4372
4373     if ($conf->{$opt}) {
4374         my $oldnet = parse_net($conf->{$opt});
4375
4376         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4377             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4378             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4379             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4380
4381             # for non online change, we try to hot-unplug
4382             die "skip\n" if !$hotplug;
4383             vm_deviceunplug($vmid, $conf, $opt);
4384         } else {
4385
4386             die "internal error" if $opt !~ m/net(\d+)/;
4387             my $iface = "tap${vmid}i$1";
4388
4389             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4390                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4391                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4392                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4393                 PVE::Network::tap_unplug($iface);
4394                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4395             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4396                 # Rate can be applied on its own but any change above needs to
4397                 # include the rate in tap_plug since OVS resets everything.
4398                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4399             }
4400
4401             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4402                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4403             }
4404
4405             return 1;
4406         }
4407     }
4408
4409     if ($hotplug) {
4410         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4411     } else {
4412         die "skip\n";
4413     }
4414 }
4415
4416 sub vmconfig_update_disk {
4417     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4418
4419     # fixme: do we need force?
4420
4421     my $drive = parse_drive($opt, $value);
4422
4423     if ($conf->{$opt}) {
4424
4425         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4426
4427             my $media = $drive->{media} || 'disk';
4428             my $oldmedia = $old_drive->{media} || 'disk';
4429             die "unable to change media type\n" if $media ne $oldmedia;
4430
4431             if (!drive_is_cdrom($old_drive)) {
4432
4433                 if ($drive->{file} ne $old_drive->{file}) {
4434
4435                     die "skip\n" if !$hotplug;
4436
4437                     # unplug and register as unused
4438                     vm_deviceunplug($vmid, $conf, $opt);
4439                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4440
4441                 } else {
4442                     # update existing disk
4443
4444                     # skip non hotpluggable value
4445                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4446                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4447                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4448                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4449                         die "skip\n";
4450                     }
4451
4452                     # apply throttle
4453                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4454                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4455                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4456                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4457                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4458                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4459                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4460                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4461                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4462                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4463                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4464                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4465                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4466                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4467                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4468                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4469                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4470                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4471
4472                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4473                                                    ($drive->{mbps} || 0)*1024*1024,
4474                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4475                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4476                                                    $drive->{iops} || 0,
4477                                                    $drive->{iops_rd} || 0,
4478                                                    $drive->{iops_wr} || 0,
4479                                                    ($drive->{mbps_max} || 0)*1024*1024,
4480                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4481                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4482                                                    $drive->{iops_max} || 0,
4483                                                    $drive->{iops_rd_max} || 0,
4484                                                    $drive->{iops_wr_max} || 0,
4485                                                    $drive->{bps_max_length} || 1,
4486                                                    $drive->{bps_rd_max_length} || 1,
4487                                                    $drive->{bps_wr_max_length} || 1,
4488                                                    $drive->{iops_max_length} || 1,
4489                                                    $drive->{iops_rd_max_length} || 1,
4490                                                    $drive->{iops_wr_max_length} || 1);
4491
4492                     }
4493
4494                     return 1;
4495                 }
4496
4497             } else { # cdrom
4498
4499                 if ($drive->{file} eq 'none') {
4500                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4501                 } else {
4502                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4503                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4504                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4505                 }
4506
4507                 return 1;
4508             }
4509         }
4510     }
4511
4512     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4513     # hotplug new disks
4514     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4515     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4516 }
4517
4518 sub vm_start {
4519     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4520         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4521
4522     PVE::QemuConfig->lock_config($vmid, sub {
4523         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4524
4525         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4526
4527         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4528
4529         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4530
4531         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4532             vmconfig_apply_pending($vmid, $conf, $storecfg);
4533             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4534         }
4535
4536         my $defaults = load_defaults();
4537
4538         # set environment variable useful inside network script
4539         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4540
4541         my $local_volumes = {};
4542
4543         if ($targetstorage) {
4544             foreach_drive($conf, sub {
4545                 my ($ds, $drive) = @_;
4546
4547                 return if drive_is_cdrom($drive);
4548
4549                 my $volid = $drive->{file};
4550
4551                 return if !$volid;
4552
4553                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4554
4555                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4556                 return if $scfg->{shared};
4557                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4558             });
4559
4560             my $format = undef;
4561
4562             foreach my $opt (sort keys %$local_volumes) {
4563
4564                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4565                 my $drive = parse_drive($opt, $conf->{$opt});
4566
4567                 #if remote storage is specified, use default format
4568                 if ($targetstorage && $targetstorage ne "1") {
4569                     $storeid = $targetstorage;
4570                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4571                     $format = $defFormat;
4572                 } else {
4573                     #else we use same format than original
4574                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4575                     $format = qemu_img_format($scfg, $volid);
4576                 }
4577
4578                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4579                 my $newdrive = $drive;
4580                 $newdrive->{format} = $format;
4581                 $newdrive->{file} = $newvolid;
4582                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4583                 $local_volumes->{$opt} = $drivestr;
4584                 #pass drive to conf for command line
4585                 $conf->{$opt} = $drivestr;
4586             }
4587         }
4588
4589         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4590
4591         my $migrate_port = 0;
4592         my $migrate_uri;
4593         if ($statefile) {
4594             if ($statefile eq 'tcp') {
4595                 my $localip = "localhost";
4596                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4597                 my $nodename = PVE::INotify::nodename();
4598
4599                 if (!defined($migration_type)) {
4600                     if (defined($datacenterconf->{migration}->{type})) {
4601                         $migration_type = $datacenterconf->{migration}->{type};
4602                     } else {
4603                         $migration_type = 'secure';
4604                     }
4605                 }
4606
4607                 if ($migration_type eq 'insecure') {
4608                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4609                     if ($migrate_network_addr) {
4610                         $localip = $migrate_network_addr;
4611                     } else {
4612                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4613                     }
4614
4615                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4616                 }
4617
4618                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4619                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4620                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4621                 push @$cmd, '-incoming', $migrate_uri;
4622                 push @$cmd, '-S';
4623
4624             } elsif ($statefile eq 'unix') {
4625                 # should be default for secure migrations as a ssh TCP forward
4626                 # tunnel is not deterministic reliable ready and fails regurarly
4627                 # to set up in time, so use UNIX socket forwards
4628                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4629                 unlink $socket_addr;
4630
4631                 $migrate_uri = "unix:$socket_addr";
4632
4633                 push @$cmd, '-incoming', $migrate_uri;
4634                 push @$cmd, '-S';
4635
4636             } else {
4637                 push @$cmd, '-loadstate', $statefile;
4638             }
4639         } elsif ($paused) {
4640             push @$cmd, '-S';
4641         }
4642
4643         # host pci devices
4644         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4645           my $d = parse_hostpci($conf->{"hostpci$i"});
4646           next if !$d;
4647           my $pcidevices = $d->{pciid};
4648           foreach my $pcidevice (@$pcidevices) {
4649                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4650
4651                 my $info = pci_device_info("0000:$pciid");
4652                 die "IOMMU not present\n" if !check_iommu_support();
4653                 die "no pci device info for device '$pciid'\n" if !$info;
4654                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4655                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4656           }
4657         }
4658
4659         PVE::Storage::activate_volumes($storecfg, $vollist);
4660
4661         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4662             my $cmd = [];
4663             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4664             eval  { run_command($cmd); };
4665         }
4666
4667         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4668                                                   : $defaults->{cpuunits};
4669
4670         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4671
4672         my %properties = (
4673             Slice => 'qemu.slice',
4674             KillMode => 'none',
4675             CPUShares => $cpuunits
4676         );
4677
4678         if (my $cpulimit = $conf->{cpulimit}) {
4679             $properties{CPUQuota} = int($cpulimit * 100);
4680         }
4681         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4682
4683         if ($conf->{hugepages}) {
4684
4685             my $code = sub {
4686                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4687                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4688
4689                 PVE::QemuServer::Memory::hugepages_mount();
4690                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4691
4692                 eval  {
4693                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4694                     run_command($cmd, %run_params);
4695                 };
4696
4697                 if (my $err = $@) {
4698                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4699                     die $err;
4700                 }
4701
4702                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4703             };
4704             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4705
4706         } else {
4707             eval  {
4708                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4709                 run_command($cmd, %run_params);
4710             };
4711         }
4712
4713         if (my $err = $@) {
4714             # deactivate volumes if start fails
4715             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4716             die "start failed: $err";
4717         }
4718
4719         print "migration listens on $migrate_uri\n" if $migrate_uri;
4720
4721         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4722             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4723             warn $@ if $@;
4724         }
4725
4726         #start nbd server for storage migration
4727         if ($targetstorage) {
4728             my $nodename = PVE::INotify::nodename();
4729             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4730             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4731             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4732             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4733
4734             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4735
4736             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4737
4738             foreach my $opt (sort keys %$local_volumes) {
4739                 my $volid = $local_volumes->{$opt};
4740                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4741                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4742                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4743             }
4744         }
4745
4746         if ($migratedfrom) {
4747             eval {
4748                 set_migration_caps($vmid);
4749             };
4750             warn $@ if $@;
4751
4752             if ($spice_port) {
4753                 print "spice listens on port $spice_port\n";
4754                 if ($spice_ticket) {
4755                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4756                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4757                 }
4758             }
4759
4760         } else {
4761             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4762                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4763                     if $conf->{balloon};
4764             }
4765
4766             foreach my $opt (keys %$conf) {
4767                 next if $opt !~  m/^net\d+$/;
4768                 my $nicconf = parse_net($conf->{$opt});
4769                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4770             }
4771         }
4772
4773         vm_mon_cmd_nocheck($vmid, 'qom-set',
4774                     path => "machine/peripheral/balloon0",
4775                     property => "guest-stats-polling-interval",
4776                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4777
4778     });
4779 }
4780
4781 sub vm_mon_cmd {
4782     my ($vmid, $execute, %params) = @_;
4783
4784     my $cmd = { execute => $execute, arguments => \%params };
4785     vm_qmp_command($vmid, $cmd);
4786 }
4787
4788 sub vm_mon_cmd_nocheck {
4789     my ($vmid, $execute, %params) = @_;
4790
4791     my $cmd = { execute => $execute, arguments => \%params };
4792     vm_qmp_command($vmid, $cmd, 1);
4793 }
4794
4795 sub vm_qmp_command {
4796     my ($vmid, $cmd, $nocheck) = @_;
4797
4798     my $res;
4799
4800     my $timeout;
4801     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4802         $timeout = $cmd->{arguments}->{timeout};
4803         delete $cmd->{arguments}->{timeout};
4804     }
4805
4806     eval {
4807         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4808         my $sname = qmp_socket($vmid);
4809         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4810             my $qmpclient = PVE::QMPClient->new();
4811
4812             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4813         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4814             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4815                 if scalar(%{$cmd->{arguments}});
4816             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4817         } else {
4818             die "unable to open monitor socket\n";
4819         }
4820     };
4821     if (my $err = $@) {
4822         syslog("err", "VM $vmid qmp command failed - $err");
4823         die $err;
4824     }
4825
4826     return $res;
4827 }
4828
4829 sub vm_human_monitor_command {
4830     my ($vmid, $cmdline) = @_;
4831
4832     my $res;
4833
4834     my $cmd = {
4835         execute => 'human-monitor-command',
4836         arguments => { 'command-line' => $cmdline},
4837     };
4838
4839     return vm_qmp_command($vmid, $cmd);
4840 }
4841
4842 sub vm_commandline {
4843     my ($storecfg, $vmid) = @_;
4844
4845     my $conf = PVE::QemuConfig->load_config($vmid);
4846
4847     my $defaults = load_defaults();
4848
4849     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4850
4851     return PVE::Tools::cmd2string($cmd);
4852 }
4853
4854 sub vm_reset {
4855     my ($vmid, $skiplock) = @_;
4856
4857     PVE::QemuConfig->lock_config($vmid, sub {
4858
4859         my $conf = PVE::QemuConfig->load_config($vmid);
4860
4861         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4862
4863         vm_mon_cmd($vmid, "system_reset");
4864     });
4865 }
4866
4867 sub get_vm_volumes {
4868     my ($conf) = @_;
4869
4870     my $vollist = [];
4871     foreach_volid($conf, sub {
4872         my ($volid, $attr) = @_;
4873
4874         return if $volid =~ m|^/|;
4875
4876         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4877         return if !$sid;
4878
4879         push @$vollist, $volid;
4880     });
4881
4882     return $vollist;
4883 }
4884
4885 sub vm_stop_cleanup {
4886     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4887
4888     eval {
4889
4890         if (!$keepActive) {
4891             my $vollist = get_vm_volumes($conf);
4892             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4893         }
4894
4895         foreach my $ext (qw(mon qmp pid vnc qga)) {
4896             unlink "/var/run/qemu-server/${vmid}.$ext";
4897         }
4898
4899         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4900     };
4901     warn $@ if $@; # avoid errors - just warn
4902 }
4903
4904 # Note: use $nockeck to skip tests if VM configuration file exists.
4905 # We need that when migration VMs to other nodes (files already moved)
4906 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4907 sub vm_stop {
4908     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4909
4910     $force = 1 if !defined($force) && !$shutdown;
4911
4912     if ($migratedfrom){
4913         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4914         kill 15, $pid if $pid;
4915         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4916         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4917         return;
4918     }
4919
4920     PVE::QemuConfig->lock_config($vmid, sub {
4921
4922         my $pid = check_running($vmid, $nocheck);
4923         return if !$pid;
4924
4925         my $conf;
4926         if (!$nocheck) {
4927             $conf = PVE::QemuConfig->load_config($vmid);
4928             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4929             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4930                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4931                 $timeout = $opts->{down} if $opts->{down};
4932             }
4933         }
4934
4935         $timeout = 60 if !defined($timeout);
4936
4937         eval {
4938             if ($shutdown) {
4939                 if (defined($conf) && $conf->{agent}) {
4940                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4941                 } else {
4942                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4943                 }
4944             } else {
4945                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4946             }
4947         };
4948         my $err = $@;
4949
4950         if (!$err) {
4951             my $count = 0;
4952             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4953                 $count++;
4954                 sleep 1;
4955             }
4956
4957             if ($count >= $timeout) {
4958                 if ($force) {
4959                     warn "VM still running - terminating now with SIGTERM\n";
4960                     kill 15, $pid;
4961                 } else {
4962                     die "VM quit/powerdown failed - got timeout\n";
4963                 }
4964             } else {
4965                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4966                 return;
4967             }
4968         } else {
4969             if ($force) {
4970                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4971                 kill 15, $pid;
4972             } else {
4973                 die "VM quit/powerdown failed\n";
4974             }
4975         }
4976
4977         # wait again
4978         $timeout = 10;
4979
4980         my $count = 0;
4981         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4982             $count++;
4983             sleep 1;
4984         }
4985
4986         if ($count >= $timeout) {
4987             warn "VM still running - terminating now with SIGKILL\n";
4988             kill 9, $pid;
4989             sleep 1;
4990         }
4991
4992         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4993    });
4994 }
4995
4996 sub vm_suspend {
4997     my ($vmid, $skiplock) = @_;
4998
4999     PVE::QemuConfig->lock_config($vmid, sub {
5000
5001         my $conf = PVE::QemuConfig->load_config($vmid);
5002
5003         PVE::QemuConfig->check_lock($conf)
5004             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5005
5006         vm_mon_cmd($vmid, "stop");
5007     });
5008 }
5009
5010 sub vm_resume {
5011     my ($vmid, $skiplock, $nocheck) = @_;
5012
5013     PVE::QemuConfig->lock_config($vmid, sub {
5014
5015         if (!$nocheck) {
5016
5017             my $conf = PVE::QemuConfig->load_config($vmid);
5018
5019             PVE::QemuConfig->check_lock($conf)
5020                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5021
5022             vm_mon_cmd($vmid, "cont");
5023
5024         } else {
5025             vm_mon_cmd_nocheck($vmid, "cont");
5026         }
5027     });
5028 }
5029
5030 sub vm_sendkey {
5031     my ($vmid, $skiplock, $key) = @_;
5032
5033     PVE::QemuConfig->lock_config($vmid, sub {
5034
5035         my $conf = PVE::QemuConfig->load_config($vmid);
5036
5037         # there is no qmp command, so we use the human monitor command
5038         vm_human_monitor_command($vmid, "sendkey $key");
5039     });
5040 }
5041
5042 sub vm_destroy {
5043     my ($storecfg, $vmid, $skiplock) = @_;
5044
5045     PVE::QemuConfig->lock_config($vmid, sub {
5046
5047         my $conf = PVE::QemuConfig->load_config($vmid);
5048
5049         if (!check_running($vmid)) {
5050             destroy_vm($storecfg, $vmid, undef, $skiplock);
5051         } else {
5052             die "VM $vmid is running - destroy failed\n";
5053         }
5054     });
5055 }
5056
5057 # pci helpers
5058
5059 sub file_write {
5060     my ($filename, $buf) = @_;
5061
5062     my $fh = IO::File->new($filename, "w");
5063     return undef if !$fh;
5064
5065     my $res = print $fh $buf;
5066
5067     $fh->close();
5068
5069     return $res;
5070 }
5071
5072 sub pci_device_info {
5073     my ($name) = @_;
5074
5075     my $res;
5076
5077     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5078     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5079
5080     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5081     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5082
5083     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5084     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5085
5086     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5087     return undef if !defined($product) || $product !~ s/^0x//;
5088
5089     $res = {
5090         name => $name,
5091         vendor => $vendor,
5092         product => $product,
5093         domain => $domain,
5094         bus => $bus,
5095         slot => $slot,
5096         func => $func,
5097         irq => $irq,
5098         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5099     };
5100
5101     return $res;
5102 }
5103
5104 sub pci_dev_reset {
5105     my ($dev) = @_;
5106
5107     my $name = $dev->{name};
5108
5109     my $fn = "$pcisysfs/devices/$name/reset";
5110
5111     return file_write($fn, "1");
5112 }
5113
5114 sub pci_dev_bind_to_vfio {
5115     my ($dev) = @_;
5116
5117     my $name = $dev->{name};
5118
5119     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5120
5121     if (!-d $vfio_basedir) {
5122         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5123     }
5124     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5125
5126     my $testdir = "$vfio_basedir/$name";
5127     return 1 if -d $testdir;
5128
5129     my $data = "$dev->{vendor} $dev->{product}";
5130     return undef if !file_write("$vfio_basedir/new_id", $data);
5131
5132     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5133     if (!file_write($fn, $name)) {
5134         return undef if -f $fn;
5135     }
5136
5137     $fn = "$vfio_basedir/bind";
5138     if (! -d $testdir) {
5139         return undef if !file_write($fn, $name);
5140     }
5141
5142     return -d $testdir;
5143 }
5144
5145 sub pci_dev_group_bind_to_vfio {
5146     my ($pciid) = @_;
5147
5148     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5149
5150     if (!-d $vfio_basedir) {
5151         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5152     }
5153     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5154
5155     # get IOMMU group devices
5156     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5157       my @devs = grep /^0000:/, readdir($D);
5158     closedir($D);
5159
5160     foreach my $pciid (@devs) {
5161         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5162
5163         # pci bridges, switches or root ports are not supported
5164         # they have a pci_bus subdirectory so skip them
5165         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5166
5167         my $info = pci_device_info($1);
5168         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5169     }
5170
5171     return 1;
5172 }
5173
5174 # vzdump restore implementaion
5175
5176 sub tar_archive_read_firstfile {
5177     my $archive = shift;
5178
5179     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5180
5181     # try to detect archive type first
5182     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5183         die "unable to open file '$archive'\n";
5184     my $firstfile = <$fh>;
5185     kill 15, $pid;
5186     close $fh;
5187
5188     die "ERROR: archive contaions no data\n" if !$firstfile;
5189     chomp $firstfile;
5190
5191     return $firstfile;
5192 }
5193
5194 sub tar_restore_cleanup {
5195     my ($storecfg, $statfile) = @_;
5196
5197     print STDERR "starting cleanup\n";
5198
5199     if (my $fd = IO::File->new($statfile, "r")) {
5200         while (defined(my $line = <$fd>)) {
5201             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5202                 my $volid = $2;
5203                 eval {
5204                     if ($volid =~ m|^/|) {
5205                         unlink $volid || die 'unlink failed\n';
5206                     } else {
5207                         PVE::Storage::vdisk_free($storecfg, $volid);
5208                     }
5209                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5210                 };
5211                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5212             } else {
5213                 print STDERR "unable to parse line in statfile - $line";
5214             }
5215         }
5216         $fd->close();
5217     }
5218 }
5219
5220 sub restore_archive {
5221     my ($archive, $vmid, $user, $opts) = @_;
5222
5223     my $format = $opts->{format};
5224     my $comp;
5225
5226     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5227         $format = 'tar' if !$format;
5228         $comp = 'gzip';
5229     } elsif ($archive =~ m/\.tar$/) {
5230         $format = 'tar' if !$format;
5231     } elsif ($archive =~ m/.tar.lzo$/) {
5232         $format = 'tar' if !$format;
5233         $comp = 'lzop';
5234     } elsif ($archive =~ m/\.vma$/) {
5235         $format = 'vma' if !$format;
5236     } elsif ($archive =~ m/\.vma\.gz$/) {
5237         $format = 'vma' if !$format;
5238         $comp = 'gzip';
5239     } elsif ($archive =~ m/\.vma\.lzo$/) {
5240         $format = 'vma' if !$format;
5241         $comp = 'lzop';
5242     } else {
5243         $format = 'vma' if !$format; # default
5244     }
5245
5246     # try to detect archive format
5247     if ($format eq 'tar') {
5248         return restore_tar_archive($archive, $vmid, $user, $opts);
5249     } else {
5250         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5251     }
5252 }
5253
5254 sub restore_update_config_line {
5255     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5256
5257     return if $line =~ m/^\#qmdump\#/;
5258     return if $line =~ m/^\#vzdump\#/;
5259     return if $line =~ m/^lock:/;
5260     return if $line =~ m/^unused\d+:/;
5261     return if $line =~ m/^parent:/;
5262     return if $line =~ m/^template:/; # restored VM is never a template
5263
5264     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5265     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5266         # try to convert old 1.X settings
5267         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5268         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5269             my ($model, $macaddr) = split(/\=/, $devconfig);
5270             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5271             my $net = {
5272                 model => $model,
5273                 bridge => "vmbr$ind",
5274                 macaddr => $macaddr,
5275             };
5276             my $netstr = print_net($net);
5277
5278             print $outfd "net$cookie->{netcount}: $netstr\n";
5279             $cookie->{netcount}++;
5280         }
5281     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5282         my ($id, $netstr) = ($1, $2);
5283         my $net = parse_net($netstr);
5284         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5285         $netstr = print_net($net);
5286         print $outfd "$id: $netstr\n";
5287     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5288         my $virtdev = $1;
5289         my $value = $3;
5290         my $di = parse_drive($virtdev, $value);
5291         if (defined($di->{backup}) && !$di->{backup}) {
5292             print $outfd "#$line";
5293         } elsif ($map->{$virtdev}) {
5294             delete $di->{format}; # format can change on restore
5295             $di->{file} = $map->{$virtdev};
5296             $value = print_drive($vmid, $di);
5297             print $outfd "$virtdev: $value\n";
5298         } else {
5299             print $outfd $line;
5300         }
5301     } else {
5302         print $outfd $line;
5303     }
5304 }
5305
5306 sub scan_volids {
5307     my ($cfg, $vmid) = @_;
5308
5309     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5310
5311     my $volid_hash = {};
5312     foreach my $storeid (keys %$info) {
5313         foreach my $item (@{$info->{$storeid}}) {
5314             next if !($item->{volid} && $item->{size});
5315             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5316             $volid_hash->{$item->{volid}} = $item;
5317         }
5318     }
5319
5320     return $volid_hash;
5321 }
5322
5323 sub is_volume_in_use {
5324     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5325
5326     my $path = PVE::Storage::path($storecfg, $volid);
5327
5328     my $scan_config = sub {
5329         my ($cref, $snapname) = @_;
5330
5331         foreach my $key (keys %$cref) {
5332             my $value = $cref->{$key};
5333             if (is_valid_drivename($key)) {
5334                 next if $skip_drive && $key eq $skip_drive;
5335                 my $drive = parse_drive($key, $value);
5336                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5337                 return 1 if $volid eq $drive->{file};
5338                 if ($drive->{file} =~ m!^/!) {
5339                     return 1 if $drive->{file} eq $path;
5340                 } else {
5341                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5342                     next if !$storeid;
5343                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5344                     next if !$scfg;
5345                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5346                 }
5347             }
5348         }
5349
5350         return 0;
5351     };
5352
5353     return 1 if &$scan_config($conf);
5354
5355     undef $skip_drive;
5356
5357     foreach my $snapname (keys %{$conf->{snapshots}}) {
5358         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5359     }
5360
5361     return 0;
5362 }
5363
5364 sub update_disksize {
5365     my ($vmid, $conf, $volid_hash) = @_;
5366
5367     my $changes;
5368
5369     # used and unused disks
5370     my $referenced = {};
5371
5372     # Note: it is allowed to define multiple storages with same path (alias), so
5373     # we need to check both 'volid' and real 'path' (two different volid can point
5374     # to the same path).
5375
5376     my $referencedpath = {};
5377
5378     # update size info
5379     foreach my $opt (keys %$conf) {
5380         if (is_valid_drivename($opt)) {
5381             my $drive = parse_drive($opt, $conf->{$opt});
5382             my $volid = $drive->{file};
5383             next if !$volid;
5384
5385             $referenced->{$volid} = 1;
5386             if ($volid_hash->{$volid} &&
5387                 (my $path = $volid_hash->{$volid}->{path})) {
5388                 $referencedpath->{$path} = 1;
5389             }
5390
5391             next if drive_is_cdrom($drive);
5392             next if !$volid_hash->{$volid};
5393
5394             $drive->{size} = $volid_hash->{$volid}->{size};
5395             my $new = print_drive($vmid, $drive);
5396             if ($new ne $conf->{$opt}) {
5397                 $changes = 1;
5398                 $conf->{$opt} = $new;
5399             }
5400         }
5401     }
5402
5403     # remove 'unusedX' entry if volume is used
5404     foreach my $opt (keys %$conf) {
5405         next if $opt !~ m/^unused\d+$/;
5406         my $volid = $conf->{$opt};
5407         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5408         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5409             $changes = 1;
5410             delete $conf->{$opt};
5411         }
5412
5413         $referenced->{$volid} = 1;
5414         $referencedpath->{$path} = 1 if $path;
5415     }
5416
5417     foreach my $volid (sort keys %$volid_hash) {
5418         next if $volid =~ m/vm-$vmid-state-/;
5419         next if $referenced->{$volid};
5420         my $path = $volid_hash->{$volid}->{path};
5421         next if !$path; # just to be sure
5422         next if $referencedpath->{$path};
5423         $changes = 1;
5424         PVE::QemuConfig->add_unused_volume($conf, $volid);
5425         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5426     }
5427
5428     return $changes;
5429 }
5430
5431 sub rescan {
5432     my ($vmid, $nolock) = @_;
5433
5434     my $cfg = PVE::Storage::config();
5435
5436     my $volid_hash = scan_volids($cfg, $vmid);
5437
5438     my $updatefn =  sub {
5439         my ($vmid) = @_;
5440
5441         my $conf = PVE::QemuConfig->load_config($vmid);
5442
5443         PVE::QemuConfig->check_lock($conf);
5444
5445         my $vm_volids = {};
5446         foreach my $volid (keys %$volid_hash) {
5447             my $info = $volid_hash->{$volid};
5448             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5449         }
5450
5451         my $changes = update_disksize($vmid, $conf, $vm_volids);
5452
5453         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5454     };
5455
5456     if (defined($vmid)) {
5457         if ($nolock) {
5458             &$updatefn($vmid);
5459         } else {
5460             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5461         }
5462     } else {
5463         my $vmlist = config_list();
5464         foreach my $vmid (keys %$vmlist) {
5465             if ($nolock) {
5466                 &$updatefn($vmid);
5467             } else {
5468                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5469             }
5470         }
5471     }
5472 }
5473
5474 sub restore_vma_archive {
5475     my ($archive, $vmid, $user, $opts, $comp) = @_;
5476
5477     my $input = $archive eq '-' ? "<&STDIN" : undef;
5478     my $readfrom = $archive;
5479
5480     my $uncomp = '';
5481     if ($comp) {
5482         $readfrom = '-';
5483         my $qarchive = PVE::Tools::shellquote($archive);
5484         if ($comp eq 'gzip') {
5485             $uncomp = "zcat $qarchive|";
5486         } elsif ($comp eq 'lzop') {
5487             $uncomp = "lzop -d -c $qarchive|";
5488         } else {
5489             die "unknown compression method '$comp'\n";
5490         }
5491
5492     }
5493
5494     my $tmpdir = "/var/tmp/vzdumptmp$$";
5495     rmtree $tmpdir;
5496
5497     # disable interrupts (always do cleanups)
5498     local $SIG{INT} =
5499         local $SIG{TERM} =
5500         local $SIG{QUIT} =
5501         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5502
5503     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5504     POSIX::mkfifo($mapfifo, 0600);
5505     my $fifofh;
5506
5507     my $openfifo = sub {
5508         open($fifofh, '>', $mapfifo) || die $!;
5509     };
5510
5511     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5512
5513     my $oldtimeout;
5514     my $timeout = 5;
5515
5516     my $devinfo = {};
5517
5518     my $rpcenv = PVE::RPCEnvironment::get();
5519
5520     my $conffile = PVE::QemuConfig->config_file($vmid);
5521     my $tmpfn = "$conffile.$$.tmp";
5522
5523     # Note: $oldconf is undef if VM does not exists
5524     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5525     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5526
5527     my $print_devmap = sub {
5528         my $virtdev_hash = {};
5529
5530         my $cfgfn = "$tmpdir/qemu-server.conf";
5531
5532         # we can read the config - that is already extracted
5533         my $fh = IO::File->new($cfgfn, "r") ||
5534             "unable to read qemu-server.conf - $!\n";
5535
5536         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5537         if (-f $fwcfgfn) {
5538             my $pve_firewall_dir = '/etc/pve/firewall';
5539             mkdir $pve_firewall_dir; # make sure the dir exists
5540             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5541         }
5542
5543         while (defined(my $line = <$fh>)) {
5544             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5545                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5546                 die "archive does not contain data for drive '$virtdev'\n"
5547                     if !$devinfo->{$devname};
5548                 if (defined($opts->{storage})) {
5549                     $storeid = $opts->{storage} || 'local';
5550                 } elsif (!$storeid) {
5551                     $storeid = 'local';
5552                 }
5553                 $format = 'raw' if !$format;
5554                 $devinfo->{$devname}->{devname} = $devname;
5555                 $devinfo->{$devname}->{virtdev} = $virtdev;
5556                 $devinfo->{$devname}->{format} = $format;
5557                 $devinfo->{$devname}->{storeid} = $storeid;
5558
5559                 # check permission on storage
5560                 my $pool = $opts->{pool}; # todo: do we need that?
5561                 if ($user ne 'root@pam') {
5562                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5563                 }
5564
5565                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5566             }
5567         }
5568
5569         foreach my $devname (keys %$devinfo) {
5570             die "found no device mapping information for device '$devname'\n"
5571                 if !$devinfo->{$devname}->{virtdev};
5572         }
5573
5574         my $cfg = PVE::Storage::config();
5575
5576         # create empty/temp config
5577         if ($oldconf) {
5578             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5579             foreach_drive($oldconf, sub {
5580                 my ($ds, $drive) = @_;
5581
5582                 return if drive_is_cdrom($drive);
5583
5584                 my $volid = $drive->{file};
5585
5586                 return if !$volid || $volid =~ m|^/|;
5587
5588                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5589                 return if !$path || !$owner || ($owner != $vmid);
5590
5591                 # Note: only delete disk we want to restore
5592                 # other volumes will become unused
5593                 if ($virtdev_hash->{$ds}) {
5594                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5595                     if (my $err = $@) {
5596                         warn $err;
5597                     }
5598                 }
5599             });
5600
5601             # delete vmstate files
5602             # since after the restore we have no snapshots anymore
5603             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5604                 my $snap = $oldconf->{snapshots}->{$snapname};
5605                 if ($snap->{vmstate}) {
5606                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5607                     if (my $err = $@) {
5608                         warn $err;
5609                     }
5610                 }
5611             }
5612         }
5613
5614         my $map = {};
5615         foreach my $virtdev (sort keys %$virtdev_hash) {
5616             my $d = $virtdev_hash->{$virtdev};
5617             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5618             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5619
5620             # test if requested format is supported
5621             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5622             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5623             $d->{format} = $defFormat if !$supported;
5624
5625             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5626                                                   $d->{format}, undef, $alloc_size);
5627             print STDERR "new volume ID is '$volid'\n";
5628             $d->{volid} = $volid;
5629             my $path = PVE::Storage::path($cfg, $volid);
5630
5631             PVE::Storage::activate_volumes($cfg,[$volid]);
5632
5633             my $write_zeros = 1;
5634             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5635                 $write_zeros = 0;
5636             }
5637
5638             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5639
5640             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5641             $map->{$virtdev} = $volid;
5642         }
5643
5644         $fh->seek(0, 0) || die "seek failed - $!\n";
5645
5646         my $outfd = new IO::File ($tmpfn, "w") ||
5647             die "unable to write config for VM $vmid\n";
5648
5649         my $cookie = { netcount => 0 };
5650         while (defined(my $line = <$fh>)) {
5651             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5652         }
5653
5654         $fh->close();
5655         $outfd->close();
5656     };
5657
5658     eval {
5659         # enable interrupts
5660         local $SIG{INT} =
5661             local $SIG{TERM} =
5662             local $SIG{QUIT} =
5663             local $SIG{HUP} =
5664             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5665         local $SIG{ALRM} = sub { die "got timeout\n"; };
5666
5667         $oldtimeout = alarm($timeout);
5668
5669         my $parser = sub {
5670             my $line = shift;
5671
5672             print "$line\n";
5673
5674             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5675                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5676                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5677             } elsif ($line =~ m/^CTIME: /) {
5678                 # we correctly received the vma config, so we can disable
5679                 # the timeout now for disk allocation (set to 10 minutes, so
5680                 # that we always timeout if something goes wrong)
5681                 alarm(600);
5682                 &$print_devmap();
5683                 print $fifofh "done\n";
5684                 my $tmp = $oldtimeout || 0;
5685                 $oldtimeout = undef;
5686                 alarm($tmp);
5687                 close($fifofh);
5688             }
5689         };
5690
5691         print "restore vma archive: $cmd\n";
5692         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5693     };
5694     my $err = $@;
5695
5696     alarm($oldtimeout) if $oldtimeout;
5697
5698     my $vollist = [];
5699     foreach my $devname (keys %$devinfo) {
5700         my $volid = $devinfo->{$devname}->{volid};
5701         push @$vollist, $volid if $volid;
5702     }
5703
5704     my $cfg = PVE::Storage::config();
5705     PVE::Storage::deactivate_volumes($cfg, $vollist);
5706
5707     unlink $mapfifo;
5708
5709     if ($err) {
5710         rmtree $tmpdir;
5711         unlink $tmpfn;
5712
5713         foreach my $devname (keys %$devinfo) {
5714             my $volid = $devinfo->{$devname}->{volid};
5715             next if !$volid;
5716             eval {
5717                 if ($volid =~ m|^/|) {
5718                     unlink $volid || die 'unlink failed\n';
5719                 } else {
5720                     PVE::Storage::vdisk_free($cfg, $volid);
5721                 }
5722                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5723             };
5724             print STDERR "unable to cleanup '$volid' - $@" if $@;
5725         }
5726         die $err;
5727     }
5728
5729     rmtree $tmpdir;
5730
5731     rename($tmpfn, $conffile) ||
5732         die "unable to commit configuration file '$conffile'\n";
5733
5734     PVE::Cluster::cfs_update(); # make sure we read new file
5735
5736     eval { rescan($vmid, 1); };
5737     warn $@ if $@;
5738 }
5739
5740 sub restore_tar_archive {
5741     my ($archive, $vmid, $user, $opts) = @_;
5742
5743     if ($archive ne '-') {
5744         my $firstfile = tar_archive_read_firstfile($archive);
5745         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5746             if $firstfile ne 'qemu-server.conf';
5747     }
5748
5749     my $storecfg = PVE::Storage::config();
5750
5751     # destroy existing data - keep empty config
5752     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5753     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5754
5755     my $tocmd = "/usr/lib/qemu-server/qmextract";
5756
5757     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5758     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5759     $tocmd .= ' --prealloc' if $opts->{prealloc};
5760     $tocmd .= ' --info' if $opts->{info};
5761
5762     # tar option "xf" does not autodetect compression when read from STDIN,
5763     # so we pipe to zcat
5764     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5765         PVE::Tools::shellquote("--to-command=$tocmd");
5766
5767     my $tmpdir = "/var/tmp/vzdumptmp$$";
5768     mkpath $tmpdir;
5769
5770     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5771     local $ENV{VZDUMP_VMID} = $vmid;
5772     local $ENV{VZDUMP_USER} = $user;
5773
5774     my $conffile = PVE::QemuConfig->config_file($vmid);
5775     my $tmpfn = "$conffile.$$.tmp";
5776
5777     # disable interrupts (always do cleanups)
5778     local $SIG{INT} =
5779         local $SIG{TERM} =
5780         local $SIG{QUIT} =
5781         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5782
5783     eval {
5784         # enable interrupts
5785         local $SIG{INT} =
5786             local $SIG{TERM} =
5787             local $SIG{QUIT} =
5788             local $SIG{HUP} =
5789             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5790
5791         if ($archive eq '-') {
5792             print "extracting archive from STDIN\n";
5793             run_command($cmd, input => "<&STDIN");
5794         } else {
5795             print "extracting archive '$archive'\n";
5796             run_command($cmd);
5797         }
5798
5799         return if $opts->{info};
5800
5801         # read new mapping
5802         my $map = {};
5803         my $statfile = "$tmpdir/qmrestore.stat";
5804         if (my $fd = IO::File->new($statfile, "r")) {
5805             while (defined (my $line = <$fd>)) {
5806                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5807                     $map->{$1} = $2 if $1;
5808                 } else {
5809                     print STDERR "unable to parse line in statfile - $line\n";
5810                 }
5811             }
5812             $fd->close();
5813         }
5814
5815         my $confsrc = "$tmpdir/qemu-server.conf";
5816
5817         my $srcfd = new IO::File($confsrc, "r") ||
5818             die "unable to open file '$confsrc'\n";
5819
5820         my $outfd = new IO::File ($tmpfn, "w") ||
5821             die "unable to write config for VM $vmid\n";
5822
5823         my $cookie = { netcount => 0 };
5824         while (defined (my $line = <$srcfd>)) {
5825             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5826         }
5827
5828         $srcfd->close();
5829         $outfd->close();
5830     };
5831     my $err = $@;
5832
5833     if ($err) {
5834
5835         unlink $tmpfn;
5836
5837         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5838
5839         die $err;
5840     }
5841
5842     rmtree $tmpdir;
5843
5844     rename $tmpfn, $conffile ||
5845         die "unable to commit configuration file '$conffile'\n";
5846
5847     PVE::Cluster::cfs_update(); # make sure we read new file
5848
5849     eval { rescan($vmid, 1); };
5850     warn $@ if $@;
5851 };
5852
5853 sub foreach_storage_used_by_vm {
5854     my ($conf, $func) = @_;
5855
5856     my $sidhash = {};
5857
5858     foreach_drive($conf, sub {
5859         my ($ds, $drive) = @_;
5860         return if drive_is_cdrom($drive);
5861
5862         my $volid = $drive->{file};
5863
5864         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5865         $sidhash->{$sid} = $sid if $sid;
5866     });
5867
5868     foreach my $sid (sort keys %$sidhash) {
5869         &$func($sid);
5870     }
5871 }
5872
5873 sub do_snapshots_with_qemu {
5874     my ($storecfg, $volid) = @_;
5875
5876     my $storage_name = PVE::Storage::parse_volume_id($volid);
5877
5878     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5879         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5880         return 1;
5881     }
5882
5883     if ($volid =~ m/\.(qcow2|qed)$/){
5884         return 1;
5885     }
5886
5887     return undef;
5888 }
5889
5890 sub qga_check_running {
5891     my ($vmid) = @_;
5892
5893     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5894     if ($@) {
5895         warn "Qemu Guest Agent is not running - $@";
5896         return 0;
5897     }
5898     return 1;
5899 }
5900
5901 sub template_create {
5902     my ($vmid, $conf, $disk) = @_;
5903
5904     my $storecfg = PVE::Storage::config();
5905
5906     foreach_drive($conf, sub {
5907         my ($ds, $drive) = @_;
5908
5909         return if drive_is_cdrom($drive);
5910         return if $disk && $ds ne $disk;
5911
5912         my $volid = $drive->{file};
5913         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5914
5915         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5916         $drive->{file} = $voliddst;
5917         $conf->{$ds} = print_drive($vmid, $drive);
5918         PVE::QemuConfig->write_config($vmid, $conf);
5919     });
5920 }
5921
5922 sub qemu_img_convert {
5923     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5924
5925     my $storecfg = PVE::Storage::config();
5926     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5927     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5928
5929     if ($src_storeid && $dst_storeid) {
5930
5931         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5932
5933         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5934         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5935
5936         my $src_format = qemu_img_format($src_scfg, $src_volname);
5937         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5938
5939         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5940         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5941
5942         my $cmd = [];
5943         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5944         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5945         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5946         if ($is_zero_initialized) {
5947             push @$cmd, "zeroinit:$dst_path";
5948         } else {
5949             push @$cmd, $dst_path;
5950         }
5951
5952         my $parser = sub {
5953             my $line = shift;
5954             if($line =~ m/\((\S+)\/100\%\)/){
5955                 my $percent = $1;
5956                 my $transferred = int($size * $percent / 100);
5957                 my $remaining = $size - $transferred;
5958
5959                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5960             }
5961
5962         };
5963
5964         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5965         my $err = $@;
5966         die "copy failed: $err" if $err;
5967     }
5968 }
5969
5970 sub qemu_img_format {
5971     my ($scfg, $volname) = @_;
5972
5973     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5974         return $1;
5975     } else {
5976         return "raw";
5977     }
5978 }
5979
5980 sub qemu_drive_mirror {
5981     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
5982
5983     $jobs = {} if !$jobs;
5984
5985     my $qemu_target;
5986     my $format;
5987     $jobs->{"drive-$drive"} = {};
5988
5989     if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
5990         my $server = $1;
5991         my $port = $2;
5992         my $exportname = $3;
5993
5994         $format = "nbd";
5995         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
5996         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
5997         my $cmd = ['socat', '-T30', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=5"];
5998
5999         my $pid = fork();
6000         if (!defined($pid)) {
6001             die "forking socat tunnel failed\n";
6002         } elsif ($pid == 0) {
6003             exec(@$cmd);
6004             warn "exec failed: $!\n";
6005             POSIX::_exit(-1);
6006         }
6007         $jobs->{"drive-$drive"}->{pid} = $pid;
6008
6009         my $timeout = 0;
6010         while (!-S $unixsocket) {
6011             die "nbd connection helper timed out\n"
6012                 if $timeout++ > 5;
6013             sleep 1;
6014         }
6015     } else {
6016         my $storecfg = PVE::Storage::config();
6017         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6018
6019         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6020
6021         $format = qemu_img_format($dst_scfg, $dst_volname);
6022
6023         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6024
6025         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6026     }
6027
6028     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6029     $opts->{format} = $format if $format;
6030
6031     print "drive mirror is starting for drive-$drive\n";
6032
6033     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6034
6035     if (my $err = $@) {
6036         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6037         die "mirroring error: $err";
6038     }
6039
6040     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6041 }
6042
6043 sub qemu_drive_mirror_monitor {
6044     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6045
6046     eval {
6047         my $err_complete = 0;
6048
6049         while (1) {
6050             die "storage migration timed out\n" if $err_complete > 300;
6051
6052             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6053
6054             my $running_mirror_jobs = {};
6055             foreach my $stat (@$stats) {
6056                 next if $stat->{type} ne 'mirror';
6057                 $running_mirror_jobs->{$stat->{device}} = $stat;
6058             }
6059
6060             my $readycounter = 0;
6061
6062             foreach my $job (keys %$jobs) {
6063
6064                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6065                     print "$job : finished\n";
6066                     delete $jobs->{$job};
6067                     next;
6068                 }
6069
6070                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6071
6072                 my $busy = $running_mirror_jobs->{$job}->{busy};
6073                 my $ready = $running_mirror_jobs->{$job}->{ready};
6074                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6075                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6076                     my $remaining = $total - $transferred;
6077                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6078
6079                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6080                 }
6081
6082                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6083             }
6084
6085             last if scalar(keys %$jobs) == 0;
6086
6087             if ($readycounter == scalar(keys %$jobs)) {
6088                 print "all mirroring jobs are ready \n";
6089                 last if $skipcomplete; #do the complete later
6090
6091                 if ($vmiddst && $vmiddst != $vmid) {
6092                     my $agent_running = $qga && qga_check_running($vmid);
6093                     if ($agent_running) {
6094                         print "freeze filesystem\n";
6095                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6096                     } else {
6097                         print "suspend vm\n";
6098                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6099                     }
6100
6101                     # if we clone a disk for a new target vm, we don't switch the disk
6102                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6103
6104                     if ($agent_running) {
6105                         print "unfreeze filesystem\n";
6106                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6107                     } else {
6108                         print "resume vm\n";
6109                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6110                     }
6111
6112                     last;
6113                 } else {
6114
6115                     foreach my $job (keys %$jobs) {
6116                         # try to switch the disk if source and destination are on the same guest
6117                         print "$job: Completing block job...\n";
6118
6119                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6120                         if ($@ =~ m/cannot be completed/) {
6121                             print "$job: Block job cannot be completed, try again.\n";
6122                             $err_complete++;
6123                         }else {
6124                             print "$job: Completed successfully.\n";
6125                             $jobs->{$job}->{complete} = 1;
6126                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6127                         }
6128                     }
6129                 }
6130             }
6131             sleep 1;
6132         }
6133     };
6134     my $err = $@;
6135
6136     if ($err) {
6137         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6138         die "mirroring error: $err";
6139     }
6140
6141 }
6142
6143 sub qemu_blockjobs_cancel {
6144     my ($vmid, $jobs) = @_;
6145
6146     foreach my $job (keys %$jobs) {
6147         print "$job: Cancelling block job\n";
6148         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6149         $jobs->{$job}->{cancel} = 1;
6150     }
6151
6152     while (1) {
6153         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6154
6155         my $running_jobs = {};
6156         foreach my $stat (@$stats) {
6157             $running_jobs->{$stat->{device}} = $stat;
6158         }
6159
6160         foreach my $job (keys %$jobs) {
6161
6162             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6163                 print "$job: Done.\n";
6164                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6165                 delete $jobs->{$job};
6166             }
6167         }
6168
6169         last if scalar(keys %$jobs) == 0;
6170
6171         sleep 1;
6172     }
6173 }
6174
6175 sub qemu_blockjobs_finish_tunnel {
6176    my ($vmid, $job, $cpid) = @_;
6177
6178    return if !$cpid;
6179
6180    for (my $i = 1; $i < 20; $i++) {
6181         my $waitpid = waitpid($cpid, WNOHANG);
6182         last if (defined($waitpid) && ($waitpid == $cpid));
6183
6184         if ($i == 10) {
6185             kill(15, $cpid);
6186         } elsif ($i >= 15) {
6187             kill(9, $cpid);
6188         }
6189         sleep (1);
6190     }
6191     unlink "/run/qemu-server/$vmid.mirror-$job";
6192 }
6193
6194 sub clone_disk {
6195     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6196         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6197
6198     my $newvolid;
6199
6200     if (!$full) {
6201         print "create linked clone of drive $drivename ($drive->{file})\n";
6202         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6203         push @$newvollist, $newvolid;
6204     } else {
6205
6206         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6207         $storeid = $storage if $storage;
6208
6209         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6210         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6211
6212         print "create full clone of drive $drivename ($drive->{file})\n";
6213         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, undef, ($size/1024));
6214         push @$newvollist, $newvolid;
6215
6216         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6217
6218         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6219         if (!$running || $snapname) {
6220             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6221         } else {
6222
6223             my $kvmver = get_running_qemu_version ($vmid);
6224             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6225                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6226                     if $drive->{iothread};
6227             }
6228
6229             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6230         }
6231     }
6232
6233     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6234
6235     my $disk = $drive;
6236     $disk->{format} = undef;
6237     $disk->{file} = $newvolid;
6238     $disk->{size} = $size;
6239
6240     return $disk;
6241 }
6242
6243 # this only works if VM is running
6244 sub get_current_qemu_machine {
6245     my ($vmid) = @_;
6246
6247     my $cmd = { execute => 'query-machines', arguments => {} };
6248     my $res = vm_qmp_command($vmid, $cmd);
6249
6250     my ($current, $default);
6251     foreach my $e (@$res) {
6252         $default = $e->{name} if $e->{'is-default'};
6253         $current = $e->{name} if $e->{'is-current'};
6254     }
6255
6256     # fallback to the default machine if current is not supported by qemu
6257     return $current || $default || 'pc';
6258 }
6259
6260 sub get_running_qemu_version {
6261     my ($vmid) = @_;
6262     my $cmd = { execute => 'query-version', arguments => {} };
6263     my $res = vm_qmp_command($vmid, $cmd);
6264     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6265 }
6266
6267 sub qemu_machine_feature_enabled {
6268     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6269
6270     my $current_major;
6271     my $current_minor;
6272
6273     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6274
6275         $current_major = $3;
6276         $current_minor = $4;
6277
6278     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6279
6280         $current_major = $1;
6281         $current_minor = $2;
6282     }
6283
6284     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6285
6286
6287 }
6288
6289 sub qemu_machine_pxe {
6290     my ($vmid, $conf, $machine) = @_;
6291
6292     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6293
6294     foreach my $opt (keys %$conf) {
6295         next if $opt !~ m/^net(\d+)$/;
6296         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6297         next if !$net;
6298         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6299         return $machine.".pxe" if $romfile =~ m/pxe/;
6300         last;
6301     }
6302
6303     return $machine;
6304 }
6305
6306 sub qemu_use_old_bios_files {
6307     my ($machine_type) = @_;
6308
6309     return if !$machine_type;
6310
6311     my $use_old_bios_files = undef;
6312
6313     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6314         $machine_type = $1;
6315         $use_old_bios_files = 1;
6316     } else {
6317         my $kvmver = kvm_user_version();
6318         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6319         # load new efi bios files on migration. So this hack is required to allow
6320         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6321         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6322         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6323     }
6324
6325     return ($use_old_bios_files, $machine_type);
6326 }
6327
6328 sub create_efidisk {
6329     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6330
6331     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6332
6333     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6334     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6335     PVE::Storage::activate_volumes($storecfg, [$volid]);
6336
6337     my $path = PVE::Storage::path($storecfg, $volid);
6338     eval {
6339         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6340     };
6341     die "Copying EFI vars image failed: $@" if $@;
6342
6343     return ($volid, $vars_size);
6344 }
6345
6346 sub lspci {
6347
6348     my $devices = {};
6349
6350     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6351             my (undef, $id, $function) = @_;
6352             my $res = { id => $id, function => $function};
6353             push @{$devices->{$id}}, $res;
6354     });
6355
6356     # Entries should be sorted by functions.
6357     foreach my $id (keys %$devices) {
6358         my $dev = $devices->{$id};
6359         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6360     }
6361
6362     return $devices;
6363 }
6364
6365 sub vm_iothreads_list {
6366     my ($vmid) = @_;
6367
6368     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6369
6370     my $iothreads = {};
6371     foreach my $iothread (@$res) {
6372         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6373     }
6374
6375     return $iothreads;
6376 }
6377
6378 sub scsihw_infos {
6379     my ($conf, $drive) = @_;
6380
6381     my $maxdev = 0;
6382
6383     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6384         $maxdev = 7;
6385     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6386         $maxdev = 1;
6387     } else {
6388         $maxdev = 256;
6389     }
6390
6391     my $controller = int($drive->{index} / $maxdev);
6392     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6393
6394     return ($maxdev, $controller, $controller_prefix);
6395 }
6396
6397 sub add_hyperv_enlightenments {
6398     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6399
6400     return if $winversion < 6;
6401     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6402
6403     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6404
6405     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6406         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6407         push @$cpuFlags , 'hv_vapic';
6408         push @$cpuFlags , 'hv_time';
6409     } else {
6410         push @$cpuFlags , 'hv_spinlocks=0xffff';
6411     }
6412
6413     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6414         push @$cpuFlags , 'hv_reset';
6415         push @$cpuFlags , 'hv_vpindex';
6416         push @$cpuFlags , 'hv_runtime';
6417     }
6418
6419     if ($winversion >= 7) {
6420         push @$cpuFlags , 'hv_relaxed';
6421     }
6422 }
6423
6424 sub windows_version {
6425     my ($ostype) = @_;
6426
6427     return 0 if !$ostype;
6428
6429     my $winversion = 0;
6430
6431     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6432         $winversion = 5;
6433     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6434         $winversion = 6;
6435     } elsif ($ostype =~ m/^win(\d+)$/) {
6436         $winversion = $1;
6437     }
6438
6439     return $winversion;
6440 }
6441
6442 sub resolve_dst_disk_format {
6443         my ($storecfg, $storeid, $src_volname, $format) = @_;
6444         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6445
6446         if (!$format) {
6447             # if no target format is specified, use the source disk format as hint
6448             if ($src_volname) {
6449                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6450                 $format = qemu_img_format($scfg, $src_volname);
6451             } else {
6452                 return $defFormat;
6453             }
6454         }
6455
6456         # test if requested format is supported - else use default
6457         my $supported = grep { $_ eq $format } @$validFormats;
6458         $format = $defFormat if !$supported;
6459         return $format;
6460 }
6461
6462 sub resolve_first_disk {
6463     my $conf = shift;
6464     my @disks = PVE::QemuServer::valid_drive_names();
6465     my $firstdisk;
6466     foreach my $ds (reverse @disks) {
6467         next if !$conf->{$ds};
6468         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6469         next if PVE::QemuServer::drive_is_cdrom($disk);
6470         $firstdisk = $ds;
6471     }
6472     return $firstdisk;
6473 }
6474
6475 sub generate_smbios1_uuid {
6476     my ($uuid, $uuid_str);
6477     UUID::generate($uuid);
6478     UUID::unparse($uuid, $uuid_str);
6479     return "uuid=$uuid_str";
6480 }
6481
6482 # bash completion helper
6483
6484 sub complete_backup_archives {
6485     my ($cmdname, $pname, $cvalue) = @_;
6486
6487     my $cfg = PVE::Storage::config();
6488
6489     my $storeid;
6490
6491     if ($cvalue =~ m/^([^:]+):/) {
6492         $storeid = $1;
6493     }
6494
6495     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6496
6497     my $res = [];
6498     foreach my $id (keys %$data) {
6499         foreach my $item (@{$data->{$id}}) {
6500             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6501             push @$res, $item->{volid} if defined($item->{volid});
6502         }
6503     }
6504
6505     return $res;
6506 }
6507
6508 my $complete_vmid_full = sub {
6509     my ($running) = @_;
6510
6511     my $idlist = vmstatus();
6512
6513     my $res = [];
6514
6515     foreach my $id (keys %$idlist) {
6516         my $d = $idlist->{$id};
6517         if (defined($running)) {
6518             next if $d->{template};
6519             next if $running && $d->{status} ne 'running';
6520             next if !$running && $d->{status} eq 'running';
6521         }
6522         push @$res, $id;
6523
6524     }
6525     return $res;
6526 };
6527
6528 sub complete_vmid {
6529     return &$complete_vmid_full();
6530 }
6531
6532 sub complete_vmid_stopped {
6533     return &$complete_vmid_full(0);
6534 }
6535
6536 sub complete_vmid_running {
6537     return &$complete_vmid_full(1);
6538 }
6539
6540 sub complete_storage {
6541
6542     my $cfg = PVE::Storage::config();
6543     my $ids = $cfg->{ids};
6544
6545     my $res = [];
6546     foreach my $sid (keys %$ids) {
6547         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6548         next if !$ids->{$sid}->{content}->{images};
6549         push @$res, $sid;
6550     }
6551
6552     return $res;
6553 }
6554
6555 sub nbd_stop {
6556     my ($vmid) = @_;
6557
6558     vm_mon_cmd($vmid, 'nbd-server-stop');
6559 }
6560
6561 1;