PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42
  43 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  44
  45 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  46
  47 # Note about locking: we use flock on the config file protect
  48 # against concurent actions.
  49 # Aditionaly, we have a 'lock' setting in the config file. This
  50 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  51 # allowed when such lock is set. But you can ignore this kind of
  52 # lock with the --skiplock flag.
  53
  54 cfs_register_file('/qemu-server/',
  55                   \&parse_vm_config,
  56                   \&write_vm_config);
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  72     type => 'string',
  73     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  74     description => "The drive's backing file's data format.",
  75     optional => 1,
  76 });
  77
  78 #no warnings 'redefine';
  79
  80 sub cgroups_write {
  81    my ($controller, $vmid, $option, $value) = @_;
  82
  83    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  84    PVE::ProcFSTools::write_proc_entry($path, $value);
  85
  86 }
  87
  88 my $nodename = PVE::INotify::nodename();
  89
  90 mkdir "/etc/pve/nodes/$nodename";
  91 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  92 mkdir $confdir;
  93
  94 my $var_run_tmpdir = "/var/run/qemu-server";
  95 mkdir $var_run_tmpdir;
  96
  97 my $lock_dir = "/var/lock/qemu-server";
  98 mkdir $lock_dir;
  99
 100 my $pcisysfs = "/sys/bus/pci";
 101
 102 my $cpu_vendor_list = {
 103     # Intel CPUs
 104     486 => 'GenuineIntel',
 105     pentium => 'GenuineIntel',
 106     pentium2  => 'GenuineIntel',
 107     pentium3  => 'GenuineIntel',
 108     coreduo => 'GenuineIntel',
 109     core2duo => 'GenuineIntel',
 110     Conroe  => 'GenuineIntel',
 111     Penryn  => 'GenuineIntel',
 112     Nehalem  => 'GenuineIntel',
 113     Westmere => 'GenuineIntel',
 114     SandyBridge => 'GenuineIntel',
 115     IvyBridge => 'GenuineIntel',
 116     Haswell => 'GenuineIntel',
 117     'Haswell-noTSX' => 'GenuineIntel',
 118     Broadwell => 'GenuineIntel',
 119     'Broadwell-noTSX' => 'GenuineIntel',
 120     'Skylake-Client' => 'GenuineIntel',
 121
 122     # AMD CPUs
 123     athlon => 'AuthenticAMD',
 124     phenom  => 'AuthenticAMD',
 125     Opteron_G1  => 'AuthenticAMD',
 126     Opteron_G2  => 'AuthenticAMD',
 127     Opteron_G3  => 'AuthenticAMD',
 128     Opteron_G4  => 'AuthenticAMD',
 129     Opteron_G5  => 'AuthenticAMD',
 130
 131     # generic types, use vendor from host node
 132     host => 'default',
 133     kvm32 => 'default',
 134     kvm64 => 'default',
 135     qemu32 => 'default',
 136     qemu64 => 'default',
 137 };
 138
 139 my $cpu_fmt = {
 140     cputype => {
 141         description => "Emulated CPU type.",
 142         type => 'string',
 143         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 144         default => 'kvm64',
 145         default_key => 1,
 146     },
 147     hidden => {
 148         description => "Do not identify as a KVM virtual machine.",
 149         type => 'boolean',
 150         optional => 1,
 151         default => 0
 152     },
 153 };
 154
 155 my $watchdog_fmt = {
 156     model => {
 157         default_key => 1,
 158         type => 'string',
 159         enum => [qw(i6300esb ib700)],
 160         description => "Watchdog type to emulate.",
 161         default => 'i6300esb',
 162         optional => 1,
 163     },
 164     action => {
 165         type => 'string',
 166         enum => [qw(reset shutdown poweroff pause debug none)],
 167         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 168         optional => 1,
 169     },
 170 };
 171 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 172
 173 my $confdesc = {
 174     onboot => {
 175         optional => 1,
 176         type => 'boolean',
 177         description => "Specifies whether a VM will be started during system bootup.",
 178         default => 0,
 179     },
 180     autostart => {
 181         optional => 1,
 182         type => 'boolean',
 183         description => "Automatic restart after crash (currently ignored).",
 184         default => 0,
 185     },
 186     hotplug => {
 187         optional => 1,
 188         type => 'string', format => 'pve-hotplug-features',
 189         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 190         default => 'network,disk,usb',
 191     },
 192     reboot => {
 193         optional => 1,
 194         type => 'boolean',
 195         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 196         default => 1,
 197     },
 198     lock => {
 199         optional => 1,
 200         type => 'string',
 201         description => "Lock/unlock the VM.",
 202         enum => [qw(migrate backup snapshot rollback)],
 203     },
 204     cpulimit => {
 205         optional => 1,
 206         type => 'number',
 207         description => "Limit of CPU usage.",
 208         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 209         minimum => 0,
 210         maximum => 128,
 211         default => 0,
 212     },
 213     cpuunits => {
 214         optional => 1,
 215         type => 'integer',
 216         description => "CPU weight for a VM.",
 217         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 218         minimum => 2,
 219         maximum => 262144,
 220         default => 1024,
 221     },
 222     memory => {
 223         optional => 1,
 224         type => 'integer',
 225         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 226         minimum => 16,
 227         default => 512,
 228     },
 229     balloon => {
 230         optional => 1,
 231         type => 'integer',
 232         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 233         minimum => 0,
 234     },
 235     shares => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 239         minimum => 0,
 240         maximum => 50000,
 241         default => 1000,
 242     },
 243     keyboard => {
 244         optional => 1,
 245         type => 'string',
 246         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.".
 247                        "It should not be necessary to set it.",
 248         enum => PVE::Tools::kvmkeymaplist(),
 249         default => undef,
 250     },
 251     name => {
 252         optional => 1,
 253         type => 'string', format => 'dns-name',
 254         description => "Set a name for the VM. Only used on the configuration web interface.",
 255     },
 256     scsihw => {
 257         optional => 1,
 258         type => 'string',
 259         description => "SCSI controller model",
 260         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 261         default => 'lsi',
 262     },
 263     description => {
 264         optional => 1,
 265         type => 'string',
 266         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 267     },
 268     ostype => {
 269         optional => 1,
 270         type => 'string',
 271         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 272         description => "Specify guest operating system.",
 273         verbose_description => <<EODESC,
 274 Specify guest operating system. This is used to enable special
 275 optimization/features for specific operating systems:
 276
 277 [horizontal]
 278 other;; unspecified OS
 279 wxp;; Microsoft Windows XP
 280 w2k;; Microsoft Windows 2000
 281 w2k3;; Microsoft Windows 2003
 282 w2k8;; Microsoft Windows 2008
 283 wvista;; Microsoft Windows Vista
 284 win7;; Microsoft Windows 7
 285 win8;; Microsoft Windows 8/2012/2012r2
 286 win10;; Microsoft Windows 10/2016
 287 l24;; Linux 2.4 Kernel
 288 l26;; Linux 2.6/3.X Kernel
 289 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 290 EODESC
 291     },
 292     boot => {
 293         optional => 1,
 294         type => 'string',
 295         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 296         pattern => '[acdn]{1,4}',
 297         default => 'cdn',
 298     },
 299     bootdisk => {
 300         optional => 1,
 301         type => 'string', format => 'pve-qm-bootdisk',
 302         description => "Enable booting from specified disk.",
 303         pattern => '(ide|sata|scsi|virtio)\d+',
 304     },
 305     smp => {
 306         optional => 1,
 307         type => 'integer',
 308         description => "The number of CPUs. Please use option -sockets instead.",
 309         minimum => 1,
 310         default => 1,
 311     },
 312     sockets => {
 313         optional => 1,
 314         type => 'integer',
 315         description => "The number of CPU sockets.",
 316         minimum => 1,
 317         default => 1,
 318     },
 319     cores => {
 320         optional => 1,
 321         type => 'integer',
 322         description => "The number of cores per socket.",
 323         minimum => 1,
 324         default => 1,
 325     },
 326     numa => {
 327         optional => 1,
 328         type => 'boolean',
 329         description => "Enable/disable NUMA.",
 330         default => 0,
 331     },
 332     hugepages => {
 333         optional => 1,
 334         type => 'string',
 335         description => "Enable/disable hugepages memory.",
 336         enum => [qw(any 2 1024)],
 337     },
 338     vcpus => {
 339         optional => 1,
 340         type => 'integer',
 341         description => "Number of hotplugged vcpus.",
 342         minimum => 1,
 343         default => 0,
 344     },
 345     acpi => {
 346         optional => 1,
 347         type => 'boolean',
 348         description => "Enable/disable ACPI.",
 349         default => 1,
 350     },
 351     agent => {
 352         optional => 1,
 353         type => 'boolean',
 354         description => "Enable/disable Qemu GuestAgent.",
 355         default => 0,
 356     },
 357     kvm => {
 358         optional => 1,
 359         type => 'boolean',
 360         description => "Enable/disable KVM hardware virtualization.",
 361         default => 1,
 362     },
 363     tdf => {
 364         optional => 1,
 365         type => 'boolean',
 366         description => "Enable/disable time drift fix.",
 367         default => 0,
 368     },
 369     localtime => {
 370         optional => 1,
 371         type => 'boolean',
 372         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 373     },
 374     freeze => {
 375         optional => 1,
 376         type => 'boolean',
 377         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 378     },
 379     vga => {
 380         optional => 1,
 381         type => 'string',
 382         description => "Select the VGA type.",
 383         verbose_description => "Select the VGA type. If you want to use high resolution" .
 384             " modes (>= 1280x1024x16) then you should use the options " .
 385             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 386             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 387             "display sever. For win* OS you can select how many independent " .
 388             "displays you want, Linux guests can add displays them self. " .
 389             "You can also run without any graphic card, using a serial device" .
 390             " as terminal.",
 391         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 392     },
 393     watchdog => {
 394         optional => 1,
 395         type => 'string', format => 'pve-qm-watchdog',
 396         description => "Create a virtual hardware watchdog device.",
 397         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 398             " (by a guest action), the watchdog must be periodically polled " .
 399             "by an agent inside the guest or else the watchdog will reset " .
 400             "the guest (or execute the respective action specified)",
 401     },
 402     startdate => {
 403         optional => 1,
 404         type => 'string',
 405         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 406         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 407         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 408         default => 'now',
 409     },
 410     startup =>  get_standard_option('pve-startup-order'),
 411     template => {
 412         optional => 1,
 413         type => 'boolean',
 414         description => "Enable/disable Template.",
 415         default => 0,
 416     },
 417     args => {
 418         optional => 1,
 419         type => 'string',
 420         description => "Arbitrary arguments passed to kvm.",
 421         verbose_description => <<EODESCR,
 422 Arbitrary arguments passed to kvm, for example:
 423
 424 args: -no-reboot -no-hpet
 425
 426 NOTE: this option is for experts only.
 427 EODESCR
 428     },
 429     tablet => {
 430         optional => 1,
 431         type => 'boolean',
 432         default => 1,
 433         description => "Enable/disable the USB tablet device.",
 434         verbose_description => "Enable/disable the USB tablet device. This device is " .
 435             "usually needed to allow absolute mouse positioning with VNC. " .
 436             "Else the mouse runs out of sync with normal VNC clients. " .
 437             "If you're running lots of console-only guests on one host, " .
 438             "you may consider disabling this to save some context switches. " .
 439             "This is turned off by default if you use spice (-vga=qxl).",
 440     },
 441     migrate_speed => {
 442         optional => 1,
 443         type => 'integer',
 444         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 445         minimum => 0,
 446         default => 0,
 447     },
 448     migrate_downtime => {
 449         optional => 1,
 450         type => 'number',
 451         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 452         minimum => 0,
 453         default => 0.1,
 454     },
 455     cdrom => {
 456         optional => 1,
 457         type => 'string', format => 'pve-qm-ide',
 458         typetext => '<volume>',
 459         description => "This is an alias for option -ide2",
 460     },
 461     cpu => {
 462         optional => 1,
 463         description => "Emulated CPU type.",
 464         type => 'string',
 465         format => $cpu_fmt,
 466     },
 467     parent => get_standard_option('pve-snapshot-name', {
 468         optional => 1,
 469         description => "Parent snapshot name. This is used internally, and should not be modified.",
 470     }),
 471     snaptime => {
 472         optional => 1,
 473         description => "Timestamp for snapshots.",
 474         type => 'integer',
 475         minimum => 0,
 476     },
 477     vmstate => {
 478         optional => 1,
 479         type => 'string', format => 'pve-volume-id',
 480         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 481     },
 482     vmstatestorage => get_standard_option('pve-storage-id', {
 483         description => "Default storage for VM state volumes/files.",
 484         optional => 1,
 485     }),
 486     machine => {
 487         description => "Specific the Qemu machine type.",
 488         type => 'string',
 489         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 490         maxLength => 40,
 491         optional => 1,
 492     },
 493     smbios1 => {
 494         description => "Specify SMBIOS type 1 fields.",
 495         type => 'string', format => 'pve-qm-smbios1',
 496         maxLength => 256,
 497         optional => 1,
 498     },
 499     protection => {
 500         optional => 1,
 501         type => 'boolean',
 502         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 503         default => 0,
 504     },
 505     bios => {
 506         optional => 1,
 507         type => 'string',
 508         enum => [ qw(seabios ovmf) ],
 509         description => "Select BIOS implementation.",
 510         default => 'seabios',
 511     },
 512 };
 513
 514 # what about other qemu settings ?
 515 #cpu => 'string',
 516 #machine => 'string',
 517 #fda => 'file',
 518 #fdb => 'file',
 519 #mtdblock => 'file',
 520 #sd => 'file',
 521 #pflash => 'file',
 522 #snapshot => 'bool',
 523 #bootp => 'file',
 524 ##tftp => 'dir',
 525 ##smb => 'dir',
 526 #kernel => 'file',
 527 #append => 'string',
 528 #initrd => 'file',
 529 ##soundhw => 'string',
 530
 531 while (my ($k, $v) = each %$confdesc) {
 532     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 533 }
 534
 535 my $MAX_IDE_DISKS = 4;
 536 my $MAX_SCSI_DISKS = 14;
 537 my $MAX_VIRTIO_DISKS = 16;
 538 my $MAX_SATA_DISKS = 6;
 539 my $MAX_USB_DEVICES = 5;
 540 my $MAX_NETS = 32;
 541 my $MAX_UNUSED_DISKS = 8;
 542 my $MAX_HOSTPCI_DEVICES = 4;
 543 my $MAX_SERIAL_PORTS = 4;
 544 my $MAX_PARALLEL_PORTS = 3;
 545 my $MAX_NUMA = 8;
 546
 547 my $numa_fmt = {
 548     cpus => {
 549         type => "string",
 550         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 551         description => "CPUs accessing this NUMA node.",
 552         format_description => "id[-id];...",
 553     },
 554     memory => {
 555         type => "number",
 556         description => "Amount of memory this NUMA node provides.",
 557         optional => 1,
 558     },
 559     hostnodes => {
 560         type => "string",
 561         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 562         description => "Host NUMA nodes to use.",
 563         format_description => "id[-id];...",
 564         optional => 1,
 565     },
 566     policy => {
 567         type => 'string',
 568         enum => [qw(preferred bind interleave)],
 569         description => "NUMA allocation policy.",
 570         optional => 1,
 571     },
 572 };
 573 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 574 my $numadesc = {
 575     optional => 1,
 576     type => 'string', format => $numa_fmt,
 577     description => "NUMA topology.",
 578 };
 579 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 580
 581 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 582     $confdesc->{"numa$i"} = $numadesc;
 583 }
 584
 585 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 586                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 587                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 588 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 589
 590 my $net_fmt_bridge_descr = <<__EOD__;
 591 Bridge to attach the network device to. The Proxmox VE standard bridge
 592 is called 'vmbr0'.
 593
 594 If you do not specify a bridge, we create a kvm user (NATed) network
 595 device, which provides DHCP and DNS services. The following addresses
 596 are used:
 597
 598  10.0.2.2   Gateway
 599  10.0.2.3   DNS Server
 600  10.0.2.4   SMB Server
 601
 602 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 603 __EOD__
 604
 605 my $net_fmt = {
 606     macaddr => {
 607         type => 'string',
 608         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 609         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 610         format_description => "XX:XX:XX:XX:XX:XX",
 611         optional => 1,
 612     },
 613     model => {
 614         type => 'string',
 615         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 616         enum => $nic_model_list,
 617         default_key => 1,
 618     },
 619     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 620     bridge => {
 621         type => 'string',
 622         description => $net_fmt_bridge_descr,
 623         format_description => 'bridge',
 624         optional => 1,
 625     },
 626     queues => {
 627         type => 'integer',
 628         minimum => 0, maximum => 16,
 629         description => 'Number of packet queues to be used on the device.',
 630         optional => 1,
 631     },
 632     rate => {
 633         type => 'number',
 634         minimum => 0,
 635         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 636         optional => 1,
 637     },
 638     tag => {
 639         type => 'integer',
 640         minimum => 1, maximum => 4094,
 641         description => 'VLAN tag to apply to packets on this interface.',
 642         optional => 1,
 643     },
 644     trunks => {
 645         type => 'string',
 646         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 647         description => 'VLAN trunks to pass through this interface.',
 648         format_description => 'vlanid[;vlanid...]',
 649         optional => 1,
 650     },
 651     firewall => {
 652         type => 'boolean',
 653         description => 'Whether this interface should be protected by the firewall.',
 654         optional => 1,
 655     },
 656     link_down => {
 657         type => 'boolean',
 658         description => 'Whether this interface should be disconnected (like pulling the plug).',
 659         optional => 1,
 660     },
 661 };
 662
 663 my $netdesc = {
 664     optional => 1,
 665     type => 'string', format => $net_fmt,
 666     description => "Specify network devices.",
 667 };
 668
 669 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 670
 671 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 672     $confdesc->{"net$i"} = $netdesc;
 673 }
 674
 675 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 676 sub verify_volume_id_or_qm_path {
 677     my ($volid, $noerr) = @_;
 678
 679     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 680         return $volid;
 681     }
 682
 683     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 684     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 685     if ($@) {
 686         return undef if $noerr;
 687         die $@;
 688     }
 689     return $volid;
 690 }
 691
 692 my $drivename_hash;
 693
 694 my %drivedesc_base = (
 695     volume => { alias => 'file' },
 696     file => {
 697         type => 'string',
 698         format => 'pve-volume-id-or-qm-path',
 699         default_key => 1,
 700         format_description => 'volume',
 701         description => "The drive's backing volume.",
 702     },
 703     media => {
 704         type => 'string',
 705         enum => [qw(cdrom disk)],
 706         description => "The drive's media type.",
 707         default => 'disk',
 708         optional => 1
 709     },
 710     cyls => {
 711         type => 'integer',
 712         description => "Force the drive's physical geometry to have a specific cylinder count.",
 713         optional => 1
 714     },
 715     heads => {
 716         type => 'integer',
 717         description => "Force the drive's physical geometry to have a specific head count.",
 718         optional => 1
 719     },
 720     secs => {
 721         type => 'integer',
 722         description => "Force the drive's physical geometry to have a specific sector count.",
 723         optional => 1
 724     },
 725     trans => {
 726         type => 'string',
 727         enum => [qw(none lba auto)],
 728         description => "Force disk geometry bios translation mode.",
 729         optional => 1,
 730     },
 731     snapshot => {
 732         type => 'boolean',
 733         description => "Whether the drive should be included when making snapshots.",
 734         optional => 1,
 735     },
 736     cache => {
 737         type => 'string',
 738         enum => [qw(none writethrough writeback unsafe directsync)],
 739         description => "The drive's cache mode",
 740         optional => 1,
 741     },
 742     format => get_standard_option('pve-qm-image-format'),
 743     size => {
 744         type => 'string',
 745         format => 'disk-size',
 746         format_description => 'DiskSize',
 747         description => "Disk size. This is purely informational and has no effect.",
 748         optional => 1,
 749     },
 750     backup => {
 751         type => 'boolean',
 752         description => "Whether the drive should be included when making backups.",
 753         optional => 1,
 754     },
 755     replicate => {
 756         type => 'boolean',
 757         description => 'Whether the drive should considered for replication jobs.',
 758         optional => 1,
 759         default => 1,
 760     },
 761     rerror => {
 762         type => 'string',
 763         enum => [qw(ignore report stop)],
 764         description => 'Read error action.',
 765         optional => 1,
 766     },
 767     werror => {
 768         type => 'string',
 769         enum => [qw(enospc ignore report stop)],
 770         description => 'Write error action.',
 771         optional => 1,
 772     },
 773     aio => {
 774         type => 'string',
 775         enum => [qw(native threads)],
 776         description => 'AIO type to use.',
 777         optional => 1,
 778     },
 779     discard => {
 780         type => 'string',
 781         enum => [qw(ignore on)],
 782         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 783         optional => 1,
 784     },
 785     detect_zeroes => {
 786         type => 'boolean',
 787         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 788         optional => 1,
 789     },
 790     serial => {
 791         type => 'string',
 792         format => 'urlencoded',
 793         format_description => 'serial',
 794         maxLength => 20*3, # *3 since it's %xx url enoded
 795         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 796         optional => 1,
 797     }
 798 );
 799
 800 my %iothread_fmt = ( iothread => {
 801         type => 'boolean',
 802         description => "Whether to use iothreads for this drive",
 803         optional => 1,
 804 });
 805
 806 my %model_fmt = (
 807     model => {
 808         type => 'string',
 809         format => 'urlencoded',
 810         format_description => 'model',
 811         maxLength => 40*3, # *3 since it's %xx url enoded
 812         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 813         optional => 1,
 814     },
 815 );
 816
 817 my %queues_fmt = (
 818     queues => {
 819         type => 'integer',
 820         description => "Number of queues.",
 821         minimum => 2,
 822         optional => 1
 823     }
 824 );
 825
 826 my %scsiblock_fmt = (
 827     scsiblock => {
 828         type => 'boolean',
 829         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 830         optional => 1,
 831         default => 0,
 832     },
 833 );
 834
 835 my $add_throttle_desc = sub {
 836     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 837     my $d = {
 838         type => $type,
 839         format_description => $unit,
 840         description => "Maximum $what in $longunit.",
 841         optional => 1,
 842     };
 843     $d->{minimum} = $minimum if defined($minimum);
 844     $drivedesc_base{$key} = $d;
 845 };
 846 # throughput: (leaky bucket)
 847 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 848 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 849 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 850 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 851 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 852 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 853 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 854 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 855 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 856
 857 # pools: (pool of IO before throttling starts taking effect)
 858 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 859 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 860 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 861 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 862 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 863 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 864
 865 # burst lengths
 866 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 867 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 868 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 869 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 870 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 871 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 872
 873 # legacy support
 874 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
 875 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
 876 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
 877 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
 878
 879 my $ide_fmt = {
 880     %drivedesc_base,
 881     %model_fmt,
 882 };
 883 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 884
 885 my $idedesc = {
 886     optional => 1,
 887     type => 'string', format => $ide_fmt,
 888     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 889 };
 890 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 891
 892 my $scsi_fmt = {
 893     %drivedesc_base,
 894     %iothread_fmt,
 895     %queues_fmt,
 896     %scsiblock_fmt,
 897 };
 898 my $scsidesc = {
 899     optional => 1,
 900     type => 'string', format => $scsi_fmt,
 901     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 902 };
 903 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 904
 905 my $sata_fmt = {
 906     %drivedesc_base,
 907 };
 908 my $satadesc = {
 909     optional => 1,
 910     type => 'string', format => $sata_fmt,
 911     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 912 };
 913 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 914
 915 my $virtio_fmt = {
 916     %drivedesc_base,
 917     %iothread_fmt,
 918 };
 919 my $virtiodesc = {
 920     optional => 1,
 921     type => 'string', format => $virtio_fmt,
 922     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 923 };
 924 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 925
 926 my $alldrive_fmt = {
 927     %drivedesc_base,
 928     %iothread_fmt,
 929     %model_fmt,
 930     %queues_fmt,
 931     %scsiblock_fmt,
 932 };
 933
 934 my $efidisk_fmt = {
 935     volume => { alias => 'file' },
 936     file => {
 937         type => 'string',
 938         format => 'pve-volume-id-or-qm-path',
 939         default_key => 1,
 940         format_description => 'volume',
 941         description => "The drive's backing volume.",
 942     },
 943     format => get_standard_option('pve-qm-image-format'),
 944     size => {
 945         type => 'string',
 946         format => 'disk-size',
 947         format_description => 'DiskSize',
 948         description => "Disk size. This is purely informational and has no effect.",
 949         optional => 1,
 950     },
 951 };
 952
 953 my $efidisk_desc = {
 954     optional => 1,
 955     type => 'string', format => $efidisk_fmt,
 956     description => "Configure a Disk for storing EFI vars",
 957 };
 958
 959 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 960
 961 my $usb_fmt = {
 962     host => {
 963         default_key => 1,
 964         type => 'string', format => 'pve-qm-usb-device',
 965         format_description => 'HOSTUSBDEVICE|spice',
 966         description => <<EODESCR,
 967 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 968
 969  'bus-port(.port)*' (decimal numbers) or
 970  'vendor_id:product_id' (hexadeciaml numbers) or
 971  'spice'
 972
 973 You can use the 'lsusb -t' command to list existing usb devices.
 974
 975 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 976
 977 The value 'spice' can be used to add a usb redirection devices for spice.
 978 EODESCR
 979     },
 980     usb3 => {
 981         optional => 1,
 982         type => 'boolean',
 983         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 984         default => 0,
 985     },
 986 };
 987
 988 my $usbdesc = {
 989     optional => 1,
 990     type => 'string', format => $usb_fmt,
 991     description => "Configure an USB device (n is 0 to 4).",
 992 };
 993 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 994
 995 # NOTE: the match-groups of this regex are used in parse_hostpci
 996 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 997 my $hostpci_fmt = {
 998     host => {
 999         default_key => 1,
1000         type => 'string',
1001         pattern => qr/$PCIRE(;$PCIRE)*/,
1002         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1003         description => <<EODESCR,
1004 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1005 of PCI virtual functions of the host. HOSTPCIID syntax is:
1006
1007 'bus:dev.func' (hexadecimal numbers)
1008
1009 You can us the 'lspci' command to list existing PCI devices.
1010 EODESCR
1011     },
1012     rombar => {
1013         type => 'boolean',
1014         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1015         optional => 1,
1016         default => 1,
1017     },
1018     romfile => {
1019         type => 'string',
1020         pattern => '[^,;]+',
1021         format_description => 'string',
1022         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1023         optional => 1,
1024     },
1025     pcie => {
1026         type => 'boolean',
1027         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1028         optional => 1,
1029         default => 0,
1030     },
1031     'x-vga' => {
1032         type => 'boolean',
1033         description =>  "Enable vfio-vga device support.",
1034         optional => 1,
1035         default => 0,
1036     },
1037 };
1038 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1039
1040 my $hostpcidesc = {
1041         optional => 1,
1042         type => 'string', format => 'pve-qm-hostpci',
1043         description => "Map host PCI devices into guest.",
1044         verbose_description =>  <<EODESCR,
1045 Map host PCI devices into guest.
1046
1047 NOTE: This option allows direct access to host hardware. So it is no longer
1048 possible to migrate such machines - use with special care.
1049
1050 CAUTION: Experimental! User reported problems with this option.
1051 EODESCR
1052 };
1053 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1054
1055 my $serialdesc = {
1056         optional => 1,
1057         type => 'string',
1058         pattern => '(/dev/.+|socket)',
1059         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1060         verbose_description =>  <<EODESCR,
1061 Create a serial device inside the VM (n is 0 to 3), and pass through a
1062 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1063 host side (use 'qm terminal' to open a terminal connection).
1064
1065 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1066
1067 CAUTION: Experimental! User reported problems with this option.
1068 EODESCR
1069 };
1070
1071 my $paralleldesc= {
1072         optional => 1,
1073         type => 'string',
1074         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1075         description =>  "Map host parallel devices (n is 0 to 2).",
1076         verbose_description =>  <<EODESCR,
1077 Map host parallel devices (n is 0 to 2).
1078
1079 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1080
1081 CAUTION: Experimental! User reported problems with this option.
1082 EODESCR
1083 };
1084
1085 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1086     $confdesc->{"parallel$i"} = $paralleldesc;
1087 }
1088
1089 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1090     $confdesc->{"serial$i"} = $serialdesc;
1091 }
1092
1093 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1094     $confdesc->{"hostpci$i"} = $hostpcidesc;
1095 }
1096
1097 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1098     $drivename_hash->{"ide$i"} = 1;
1099     $confdesc->{"ide$i"} = $idedesc;
1100 }
1101
1102 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1103     $drivename_hash->{"sata$i"} = 1;
1104     $confdesc->{"sata$i"} = $satadesc;
1105 }
1106
1107 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1108     $drivename_hash->{"scsi$i"} = 1;
1109     $confdesc->{"scsi$i"} = $scsidesc ;
1110 }
1111
1112 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1113     $drivename_hash->{"virtio$i"} = 1;
1114     $confdesc->{"virtio$i"} = $virtiodesc;
1115 }
1116
1117 $drivename_hash->{efidisk0} = 1;
1118 $confdesc->{efidisk0} = $efidisk_desc;
1119
1120 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1121     $confdesc->{"usb$i"} = $usbdesc;
1122 }
1123
1124 my $unuseddesc = {
1125     optional => 1,
1126     type => 'string', format => 'pve-volume-id',
1127     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1128 };
1129
1130 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1131     $confdesc->{"unused$i"} = $unuseddesc;
1132 }
1133
1134 my $kvm_api_version = 0;
1135
1136 sub kvm_version {
1137
1138     return $kvm_api_version if $kvm_api_version;
1139
1140     my $fh = IO::File->new("</dev/kvm") ||
1141         return 0;
1142
1143     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1144         $kvm_api_version = $v;
1145     }
1146
1147     $fh->close();
1148
1149     return  $kvm_api_version;
1150 }
1151
1152 my $kvm_user_version;
1153
1154 sub kvm_user_version {
1155
1156     return $kvm_user_version if $kvm_user_version;
1157
1158     $kvm_user_version = 'unknown';
1159
1160     my $code = sub {
1161         my $line = shift;
1162         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1163             $kvm_user_version = $2;
1164         }
1165     };
1166
1167     eval { run_command("kvm -version", outfunc => $code); };
1168     warn $@ if $@;
1169
1170     return $kvm_user_version;
1171
1172 }
1173
1174 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1175
1176 sub valid_drive_names {
1177     # order is important - used to autoselect boot disk
1178     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1179             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1180             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1181             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1182             'efidisk0');
1183 }
1184
1185 sub is_valid_drivename {
1186     my $dev = shift;
1187
1188     return defined($drivename_hash->{$dev});
1189 }
1190
1191 sub option_exists {
1192     my $key = shift;
1193     return defined($confdesc->{$key});
1194 }
1195
1196 sub nic_models {
1197     return $nic_model_list;
1198 }
1199
1200 sub os_list_description {
1201
1202     return {
1203         other => 'Other',
1204         wxp => 'Windows XP',
1205         w2k => 'Windows 2000',
1206         w2k3 =>, 'Windows 2003',
1207         w2k8 => 'Windows 2008',
1208         wvista => 'Windows Vista',
1209         win7 => 'Windows 7',
1210         win8 => 'Windows 8/2012',
1211         win10 => 'Windows 10/2016',
1212         l24 => 'Linux 2.4',
1213         l26 => 'Linux 2.6',
1214     };
1215 }
1216
1217 my $cdrom_path;
1218
1219 sub get_cdrom_path {
1220
1221     return  $cdrom_path if $cdrom_path;
1222
1223     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1224     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1225     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1226 }
1227
1228 sub get_iso_path {
1229     my ($storecfg, $vmid, $cdrom) = @_;
1230
1231     if ($cdrom eq 'cdrom') {
1232         return get_cdrom_path();
1233     } elsif ($cdrom eq 'none') {
1234         return '';
1235     } elsif ($cdrom =~ m|^/|) {
1236         return $cdrom;
1237     } else {
1238         return PVE::Storage::path($storecfg, $cdrom);
1239     }
1240 }
1241
1242 # try to convert old style file names to volume IDs
1243 sub filename_to_volume_id {
1244     my ($vmid, $file, $media) = @_;
1245
1246     if (!($file eq 'none' || $file eq 'cdrom' ||
1247           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1248
1249         return undef if $file =~ m|/|;
1250
1251         if ($media && $media eq 'cdrom') {
1252             $file = "local:iso/$file";
1253         } else {
1254             $file = "local:$vmid/$file";
1255         }
1256     }
1257
1258     return $file;
1259 }
1260
1261 sub verify_media_type {
1262     my ($opt, $vtype, $media) = @_;
1263
1264     return if !$media;
1265
1266     my $etype;
1267     if ($media eq 'disk') {
1268         $etype = 'images';
1269     } elsif ($media eq 'cdrom') {
1270         $etype = 'iso';
1271     } else {
1272         die "internal error";
1273     }
1274
1275     return if ($vtype eq $etype);
1276
1277     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1278 }
1279
1280 sub cleanup_drive_path {
1281     my ($opt, $storecfg, $drive) = @_;
1282
1283     # try to convert filesystem paths to volume IDs
1284
1285     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1286         ($drive->{file} !~ m|^/dev/.+|) &&
1287         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1288         ($drive->{file} !~ m/^\d+$/)) {
1289         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1290         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1291         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1292         verify_media_type($opt, $vtype, $drive->{media});
1293         $drive->{file} = $volid;
1294     }
1295
1296     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1297 }
1298
1299 sub parse_hotplug_features {
1300     my ($data) = @_;
1301
1302     my $res = {};
1303
1304     return $res if $data eq '0';
1305
1306     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1307
1308     foreach my $feature (PVE::Tools::split_list($data)) {
1309         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1310             $res->{$1} = 1;
1311         } else {
1312             die "invalid hotplug feature '$feature'\n";
1313         }
1314     }
1315     return $res;
1316 }
1317
1318 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1319 sub pve_verify_hotplug_features {
1320     my ($value, $noerr) = @_;
1321
1322     return $value if parse_hotplug_features($value);
1323
1324     return undef if $noerr;
1325
1326     die "unable to parse hotplug option\n";
1327 }
1328
1329 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1330 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1331 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1332 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1333 #        [,iothread=on][,serial=serial][,model=model]
1334
1335 sub parse_drive {
1336     my ($key, $data) = @_;
1337
1338     my ($interface, $index);
1339
1340     if ($key =~ m/^([^\d]+)(\d+)$/) {
1341         $interface = $1;
1342         $index = $2;
1343     } else {
1344         return undef;
1345     }
1346
1347     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1348                                      : $confdesc->{$key}->{format};
1349     if (!$desc) {
1350         warn "invalid drive key: $key\n";
1351         return undef;
1352     }
1353     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1354     return undef if !$res;
1355     $res->{interface} = $interface;
1356     $res->{index} = $index;
1357
1358     my $error = 0;
1359     foreach my $opt (qw(bps bps_rd bps_wr)) {
1360         if (my $bps = defined(delete $res->{$opt})) {
1361             if (defined($res->{"m$opt"})) {
1362                 warn "both $opt and m$opt specified\n";
1363                 ++$error;
1364                 next;
1365             }
1366             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1367         }
1368     }
1369
1370     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1371     for my $requirement (
1372         [mbps_max => 'mbps'],
1373         [mbps_rd_max => 'mbps_rd'],
1374         [mbps_wr_max => 'mbps_wr'],
1375         [miops_max => 'miops'],
1376         [miops_rd_max => 'miops_rd'],
1377         [miops_wr_max => 'miops_wr'],
1378         [bps_max_length => 'mbps_max'],
1379         [bps_rd_max_length => 'mbps_rd_max'],
1380         [bps_wr_max_length => 'mbps_wr_max'],
1381         [iops_max_length => 'iops_max'],
1382         [iops_rd_max_length => 'iops_rd_max'],
1383         [iops_wr_max_length => 'iops_wr_max']) {
1384         my ($option, $requires) = @$requirement;
1385         if ($res->{$option} && !$res->{$requires}) {
1386             warn "$option requires $requires\n";
1387             ++$error;
1388         }
1389     }
1390
1391     return undef if $error;
1392
1393     return undef if $res->{mbps_rd} && $res->{mbps};
1394     return undef if $res->{mbps_wr} && $res->{mbps};
1395     return undef if $res->{iops_rd} && $res->{iops};
1396     return undef if $res->{iops_wr} && $res->{iops};
1397
1398     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1399         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1400         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1401         return undef if $res->{interface} eq 'virtio';
1402     }
1403
1404     if (my $size = $res->{size}) {
1405         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1406     }
1407
1408     return $res;
1409 }
1410
1411 sub print_drive {
1412     my ($vmid, $drive) = @_;
1413     my $data = { %$drive };
1414     delete $data->{$_} for qw(index interface);
1415     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1416 }
1417
1418 sub scsi_inquiry {
1419     my($fh, $noerr) = @_;
1420
1421     my $SG_IO = 0x2285;
1422     my $SG_GET_VERSION_NUM = 0x2282;
1423
1424     my $versionbuf = "\x00" x 8;
1425     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1426     if (!$ret) {
1427         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1428         return undef;
1429     }
1430     my $version = unpack("I", $versionbuf);
1431     if ($version < 30000) {
1432         die "scsi generic interface too old\n"  if !$noerr;
1433         return undef;
1434     }
1435
1436     my $buf = "\x00" x 36;
1437     my $sensebuf = "\x00" x 8;
1438     my $cmd = pack("C x3 C x1", 0x12, 36);
1439
1440     # see /usr/include/scsi/sg.h
1441     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1442
1443     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1444                       length($sensebuf), 0, length($buf), $buf,
1445                       $cmd, $sensebuf, 6000);
1446
1447     $ret = ioctl($fh, $SG_IO, $packet);
1448     if (!$ret) {
1449         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1450         return undef;
1451     }
1452
1453     my @res = unpack($sg_io_hdr_t, $packet);
1454     if ($res[17] || $res[18]) {
1455         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1456         return undef;
1457     }
1458
1459     my $res = {};
1460     (my $byte0, my $byte1, $res->{vendor},
1461      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1462
1463     $res->{removable} = $byte1 & 128 ? 1 : 0;
1464     $res->{type} = $byte0 & 31;
1465
1466     return $res;
1467 }
1468
1469 sub path_is_scsi {
1470     my ($path) = @_;
1471
1472     my $fh = IO::File->new("+<$path") || return undef;
1473     my $res = scsi_inquiry($fh, 1);
1474     close($fh);
1475
1476     return $res;
1477 }
1478
1479 sub machine_type_is_q35 {
1480     my ($conf) = @_;
1481
1482     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1483 }
1484
1485 sub print_tabletdevice_full {
1486     my ($conf) = @_;
1487
1488     my $q35 = machine_type_is_q35($conf);
1489
1490     # we use uhci for old VMs because tablet driver was buggy in older qemu
1491     my $usbbus = $q35 ? "ehci" : "uhci";
1492
1493     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1494 }
1495
1496 sub print_drivedevice_full {
1497     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1498
1499     my $device = '';
1500     my $maxdev = 0;
1501
1502     if ($drive->{interface} eq 'virtio') {
1503         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1504         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1505         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1506     } elsif ($drive->{interface} eq 'scsi') {
1507
1508         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1509         my $unit = $drive->{index} % $maxdev;
1510         my $devicetype = 'hd';
1511         my $path = '';
1512         if (drive_is_cdrom($drive)) {
1513             $devicetype = 'cd';
1514         } else {
1515             if ($drive->{file} =~ m|^/|) {
1516                 $path = $drive->{file};
1517                 if (my $info = path_is_scsi($path)) {
1518                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1519                         $devicetype = 'block';
1520                     } elsif ($info->{type} == 1) { # tape
1521                         $devicetype = 'generic';
1522                     }
1523                 }
1524             } else {
1525                  $path = PVE::Storage::path($storecfg, $drive->{file});
1526             }
1527
1528             if($path =~ m/^iscsi\:\/\//){
1529                 $devicetype = 'generic';
1530             }
1531         }
1532
1533         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1534             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1535         } else {
1536             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1537         }
1538
1539     } elsif ($drive->{interface} eq 'ide'){
1540         $maxdev = 2;
1541         my $controller = int($drive->{index} / $maxdev);
1542         my $unit = $drive->{index} % $maxdev;
1543         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1544
1545         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1546         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1547             $model = URI::Escape::uri_unescape($model);
1548             $device .= ",model=$model";
1549         }
1550     } elsif ($drive->{interface} eq 'sata'){
1551         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1552         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1553         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1554     } elsif ($drive->{interface} eq 'usb') {
1555         die "implement me";
1556         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1557     } else {
1558         die "unsupported interface type";
1559     }
1560
1561     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1562
1563     return $device;
1564 }
1565
1566 sub get_initiator_name {
1567     my $initiator;
1568
1569     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1570     while (defined(my $line = <$fh>)) {
1571         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1572         $initiator = $1;
1573         last;
1574     }
1575     $fh->close();
1576
1577     return $initiator;
1578 }
1579
1580 sub print_drive_full {
1581     my ($storecfg, $vmid, $drive) = @_;
1582
1583     my $path;
1584     my $volid = $drive->{file};
1585     my $format;
1586
1587     if (drive_is_cdrom($drive)) {
1588         $path = get_iso_path($storecfg, $vmid, $volid);
1589     } else {
1590         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1591         if ($storeid) {
1592             $path = PVE::Storage::path($storecfg, $volid);
1593             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1594             $format = qemu_img_format($scfg, $volname);
1595         } else {
1596             $path = $volid;
1597             $format = "raw";
1598         }
1599    }
1600
1601     my $opts = '';
1602     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard);
1603     foreach my $o (@qemu_drive_options) {
1604         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1605     }
1606     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1607         my ($dir, $qmpname) = @$type;
1608         if (my $v = $drive->{"mbps$dir"}) {
1609             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1610         }
1611         if (my $v = $drive->{"mbps${dir}_max"}) {
1612             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1613         }
1614         if (my $v = $drive->{"bps${dir}_max_length"}) {
1615             $opts .= ",throttling.bps$qmpname-max-length=$v";
1616         }
1617         if (my $v = $drive->{"iops${dir}"}) {
1618             $opts .= ",throttling.iops$qmpname=$v";
1619         }
1620         if (my $v = $drive->{"iops${dir}_max"}) {
1621             $opts .= ",throttling.iops$qmpname-max=$v";
1622         }
1623         if (my $v = $drive->{"iops${dir}_max_length"}) {
1624             $opts .= ",throttling.iops$qmpname-max-length=$v";
1625         }
1626     }
1627
1628     if (my $serial = $drive->{serial}) {
1629         $serial = URI::Escape::uri_unescape($serial);
1630         $opts .= ",serial=$serial";
1631     }
1632
1633     $opts .= ",format=$format" if $format && !$drive->{format};
1634
1635     my $cache_direct = 0;
1636
1637     if (my $cache = $drive->{cache}) {
1638         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1639     } elsif (!drive_is_cdrom($drive)) {
1640         $opts .= ",cache=none";
1641         $cache_direct = 1;
1642     }
1643
1644     # aio native works only with O_DIRECT
1645     if (!$drive->{aio}) {
1646         if($cache_direct) {
1647             $opts .= ",aio=native";
1648         } else {
1649             $opts .= ",aio=threads";
1650         }
1651     }
1652
1653     if (!drive_is_cdrom($drive)) {
1654         my $detectzeroes;
1655         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1656             $detectzeroes = 'off';
1657         } elsif ($drive->{discard}) {
1658             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1659         } else {
1660             # This used to be our default with discard not being specified:
1661             $detectzeroes = 'on';
1662         }
1663         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1664     }
1665
1666     my $pathinfo = $path ? "file=$path," : '';
1667
1668     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1669 }
1670
1671 sub print_netdevice_full {
1672     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1673
1674     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1675
1676     my $device = $net->{model};
1677     if ($net->{model} eq 'virtio') {
1678          $device = 'virtio-net-pci';
1679      };
1680
1681     my $pciaddr = print_pci_addr("$netid", $bridges);
1682     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1683     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1684         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1685         my $vectors = $net->{queues} * 2 + 2;
1686         $tmpstr .= ",vectors=$vectors,mq=on";
1687     }
1688     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1689
1690     if ($use_old_bios_files) {
1691         my $romfile;
1692         if ($device eq 'virtio-net-pci') {
1693             $romfile = 'pxe-virtio.rom';
1694         } elsif ($device eq 'e1000') {
1695             $romfile = 'pxe-e1000.rom';
1696         } elsif ($device eq 'ne2k') {
1697             $romfile = 'pxe-ne2k_pci.rom';
1698         } elsif ($device eq 'pcnet') {
1699             $romfile = 'pxe-pcnet.rom';
1700         } elsif ($device eq 'rtl8139') {
1701             $romfile = 'pxe-rtl8139.rom';
1702         }
1703         $tmpstr .= ",romfile=$romfile" if $romfile;
1704     }
1705
1706     return $tmpstr;
1707 }
1708
1709 sub print_netdev_full {
1710     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1711
1712     my $i = '';
1713     if ($netid =~ m/^net(\d+)$/) {
1714         $i = int($1);
1715     }
1716
1717     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1718
1719     my $ifname = "tap${vmid}i$i";
1720
1721     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1722     die "interface name '$ifname' is too long (max 15 character)\n"
1723         if length($ifname) >= 16;
1724
1725     my $vhostparam = '';
1726     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1727
1728     my $vmname = $conf->{name} || "vm$vmid";
1729
1730     my $netdev = "";
1731     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1732
1733     if ($net->{bridge}) {
1734         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1735     } else {
1736         $netdev = "type=user,id=$netid,hostname=$vmname";
1737     }
1738
1739     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1740
1741     return $netdev;
1742 }
1743
1744
1745 sub print_cpu_device {
1746     my ($conf, $id) = @_;
1747
1748     my $kvm = $conf->{kvm} // 1;
1749     my $cpu = $kvm ? "kvm64" : "qemu64";
1750     if (my $cputype = $conf->{cpu}) {
1751         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1752             or die "Cannot parse cpu description: $cputype\n";
1753         $cpu = $cpuconf->{cputype};
1754     }
1755
1756     my $cores = $conf->{cores} || 1;
1757
1758     my $current_core = ($id - 1) % $cores;
1759     my $current_socket = int(($id - 1 - $current_core)/$cores);
1760
1761     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1762 }
1763
1764 sub drive_is_cdrom {
1765     my ($drive) = @_;
1766
1767     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1768
1769 }
1770
1771 sub parse_number_sets {
1772     my ($set) = @_;
1773     my $res = [];
1774     foreach my $part (split(/;/, $set)) {
1775         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1776             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1777             push @$res, [ $1, $2 ];
1778         } else {
1779             die "invalid range: $part\n";
1780         }
1781     }
1782     return $res;
1783 }
1784
1785 sub parse_numa {
1786     my ($data) = @_;
1787
1788     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1789     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1790     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1791     return $res;
1792 }
1793
1794 sub parse_hostpci {
1795     my ($value) = @_;
1796
1797     return undef if !$value;
1798
1799     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1800
1801     my @idlist = split(/;/, $res->{host});
1802     delete $res->{host};
1803     foreach my $id (@idlist) {
1804         if ($id =~ /^$PCIRE$/) {
1805             if (defined($2)) {
1806                 push @{$res->{pciid}}, { id => $1, function => $2 };
1807             } else {
1808                 my $pcidevices = lspci($1);
1809                 $res->{pciid} = $pcidevices->{$1};
1810             }
1811         } else {
1812             # should have been caught by parse_property_string already
1813             die "failed to parse PCI id: $id\n";
1814         }
1815     }
1816     return $res;
1817 }
1818
1819 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1820 sub parse_net {
1821     my ($data) = @_;
1822
1823     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1824     if ($@) {
1825         warn $@;
1826         return undef;
1827     }
1828     if (!defined($res->{macaddr})) {
1829         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1830         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1831     }
1832     return $res;
1833 }
1834
1835 sub print_net {
1836     my $net = shift;
1837
1838     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1839 }
1840
1841 sub add_random_macs {
1842     my ($settings) = @_;
1843
1844     foreach my $opt (keys %$settings) {
1845         next if $opt !~ m/^net(\d+)$/;
1846         my $net = parse_net($settings->{$opt});
1847         next if !$net;
1848         $settings->{$opt} = print_net($net);
1849     }
1850 }
1851
1852 sub vm_is_volid_owner {
1853     my ($storecfg, $vmid, $volid) = @_;
1854
1855     if ($volid !~  m|^/|) {
1856         my ($path, $owner);
1857         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1858         if ($owner && ($owner == $vmid)) {
1859             return 1;
1860         }
1861     }
1862
1863     return undef;
1864 }
1865
1866 sub split_flagged_list {
1867     my $text = shift || '';
1868     $text =~ s/[,;]/ /g;
1869     $text =~ s/^\s+//;
1870     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1871 }
1872
1873 sub join_flagged_list {
1874     my ($how, $lst) = @_;
1875     join $how, map { $lst->{$_} . $_ } keys %$lst;
1876 }
1877
1878 sub vmconfig_delete_pending_option {
1879     my ($conf, $key, $force) = @_;
1880
1881     delete $conf->{pending}->{$key};
1882     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1883     $pending_delete_hash->{$key} = $force ? '!' : '';
1884     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1885 }
1886
1887 sub vmconfig_undelete_pending_option {
1888     my ($conf, $key) = @_;
1889
1890     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1891     delete $pending_delete_hash->{$key};
1892
1893     if (%$pending_delete_hash) {
1894         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1895     } else {
1896         delete $conf->{pending}->{delete};
1897     }
1898 }
1899
1900 sub vmconfig_register_unused_drive {
1901     my ($storecfg, $vmid, $conf, $drive) = @_;
1902
1903     if (!drive_is_cdrom($drive)) {
1904         my $volid = $drive->{file};
1905         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1906             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1907         }
1908     }
1909 }
1910
1911 sub vmconfig_cleanup_pending {
1912     my ($conf) = @_;
1913
1914     # remove pending changes when nothing changed
1915     my $changes;
1916     foreach my $opt (keys %{$conf->{pending}}) {
1917         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1918             $changes = 1;
1919             delete $conf->{pending}->{$opt};
1920         }
1921     }
1922
1923     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1924     my $pending_delete_hash = {};
1925     while (my ($opt, $force) = each %$current_delete_hash) {
1926         if (defined($conf->{$opt})) {
1927             $pending_delete_hash->{$opt} = $force;
1928         } else {
1929             $changes = 1;
1930         }
1931     }
1932
1933     if (%$pending_delete_hash) {
1934         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1935     } else {
1936         delete $conf->{pending}->{delete};
1937     }
1938
1939     return $changes;
1940 }
1941
1942 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1943 my $smbios1_fmt = {
1944     uuid => {
1945         type => 'string',
1946         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1947         format_description => 'UUID',
1948         description => "Set SMBIOS1 UUID.",
1949         optional => 1,
1950     },
1951     version => {
1952         type => 'string',
1953         pattern => '\S+',
1954         format_description => 'string',
1955         description => "Set SMBIOS1 version.",
1956         optional => 1,
1957     },
1958     serial => {
1959         type => 'string',
1960         pattern => '\S+',
1961         format_description => 'string',
1962         description => "Set SMBIOS1 serial number.",
1963         optional => 1,
1964     },
1965     manufacturer => {
1966         type => 'string',
1967         pattern => '\S+',
1968         format_description => 'string',
1969         description => "Set SMBIOS1 manufacturer.",
1970         optional => 1,
1971     },
1972     product => {
1973         type => 'string',
1974         pattern => '\S+',
1975         format_description => 'string',
1976         description => "Set SMBIOS1 product ID.",
1977         optional => 1,
1978     },
1979     sku => {
1980         type => 'string',
1981         pattern => '\S+',
1982         format_description => 'string',
1983         description => "Set SMBIOS1 SKU string.",
1984         optional => 1,
1985     },
1986     family => {
1987         type => 'string',
1988         pattern => '\S+',
1989         format_description => 'string',
1990         description => "Set SMBIOS1 family string.",
1991         optional => 1,
1992     },
1993 };
1994
1995 sub parse_smbios1 {
1996     my ($data) = @_;
1997
1998     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1999     warn $@ if $@;
2000     return $res;
2001 }
2002
2003 sub print_smbios1 {
2004     my ($smbios1) = @_;
2005     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2006 }
2007
2008 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2009
2010 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2011 sub verify_bootdisk {
2012     my ($value, $noerr) = @_;
2013
2014     return $value if is_valid_drivename($value);
2015
2016     return undef if $noerr;
2017
2018     die "invalid boot disk '$value'\n";
2019 }
2020
2021 sub parse_watchdog {
2022     my ($value) = @_;
2023
2024     return undef if !$value;
2025
2026     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2027     warn $@ if $@;
2028     return $res;
2029 }
2030
2031 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2032 sub verify_usb_device {
2033     my ($value, $noerr) = @_;
2034
2035     return $value if parse_usb_device($value);
2036
2037     return undef if $noerr;
2038
2039     die "unable to parse usb device\n";
2040 }
2041
2042 # add JSON properties for create and set function
2043 sub json_config_properties {
2044     my $prop = shift;
2045
2046     foreach my $opt (keys %$confdesc) {
2047         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2048         $prop->{$opt} = $confdesc->{$opt};
2049     }
2050
2051     return $prop;
2052 }
2053
2054 sub check_type {
2055     my ($key, $value) = @_;
2056
2057     die "unknown setting '$key'\n" if !$confdesc->{$key};
2058
2059     my $type = $confdesc->{$key}->{type};
2060
2061     if (!defined($value)) {
2062         die "got undefined value\n";
2063     }
2064
2065     if ($value =~ m/[\n\r]/) {
2066         die "property contains a line feed\n";
2067     }
2068
2069     if ($type eq 'boolean') {
2070         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2071         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2072         die "type check ('boolean') failed - got '$value'\n";
2073     } elsif ($type eq 'integer') {
2074         return int($1) if $value =~ m/^(\d+)$/;
2075         die "type check ('integer') failed - got '$value'\n";
2076     } elsif ($type eq 'number') {
2077         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2078         die "type check ('number') failed - got '$value'\n";
2079     } elsif ($type eq 'string') {
2080         if (my $fmt = $confdesc->{$key}->{format}) {
2081             PVE::JSONSchema::check_format($fmt, $value);
2082             return $value;
2083         }
2084         $value =~ s/^\"(.*)\"$/$1/;
2085         return $value;
2086     } else {
2087         die "internal error"
2088     }
2089 }
2090
2091 sub check_iommu_support{
2092     #fixme : need to check IOMMU support
2093     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2094
2095     my $iommu=1;
2096     return $iommu;
2097
2098 }
2099
2100 sub touch_config {
2101     my ($vmid) = @_;
2102
2103     my $conf = PVE::QemuConfig->config_file($vmid);
2104     utime undef, undef, $conf;
2105 }
2106
2107 sub destroy_vm {
2108     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2109
2110     my $conffile = PVE::QemuConfig->config_file($vmid);
2111
2112     my $conf = PVE::QemuConfig->load_config($vmid);
2113
2114     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2115
2116     if ($conf->{template}) {
2117         # check if any base image is still used by a linked clone
2118         foreach_drive($conf, sub {
2119                 my ($ds, $drive) = @_;
2120
2121                 return if drive_is_cdrom($drive);
2122
2123                 my $volid = $drive->{file};
2124
2125                 return if !$volid || $volid =~ m|^/|;
2126
2127                 die "base volume '$volid' is still in use by linked cloned\n"
2128                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2129
2130         });
2131     }
2132
2133     # only remove disks owned by this VM
2134     foreach_drive($conf, sub {
2135         my ($ds, $drive) = @_;
2136
2137         return if drive_is_cdrom($drive);
2138
2139         my $volid = $drive->{file};
2140
2141         return if !$volid || $volid =~ m|^/|;
2142
2143         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2144         return if !$path || !$owner || ($owner != $vmid);
2145
2146         eval {
2147             PVE::Storage::vdisk_free($storecfg, $volid);
2148         };
2149         warn "Could not remove disk '$volid', check manually: $@" if $@;
2150
2151     });
2152
2153     if ($keep_empty_config) {
2154         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2155     } else {
2156         unlink $conffile;
2157     }
2158
2159     # also remove unused disk
2160     eval {
2161         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2162
2163         eval {
2164             PVE::Storage::foreach_volid($dl, sub {
2165                 my ($volid, $sid, $volname, $d) = @_;
2166                 PVE::Storage::vdisk_free($storecfg, $volid);
2167             });
2168         };
2169         warn $@ if $@;
2170
2171     };
2172     warn $@ if $@;
2173 }
2174
2175 sub parse_vm_config {
2176     my ($filename, $raw) = @_;
2177
2178     return undef if !defined($raw);
2179
2180     my $res = {
2181         digest => Digest::SHA::sha1_hex($raw),
2182         snapshots => {},
2183         pending => {},
2184     };
2185
2186     $filename =~ m|/qemu-server/(\d+)\.conf$|
2187         || die "got strange filename '$filename'";
2188
2189     my $vmid = $1;
2190
2191     my $conf = $res;
2192     my $descr;
2193     my $section = '';
2194
2195     my @lines = split(/\n/, $raw);
2196     foreach my $line (@lines) {
2197         next if $line =~ m/^\s*$/;
2198
2199         if ($line =~ m/^\[PENDING\]\s*$/i) {
2200             $section = 'pending';
2201             if (defined($descr)) {
2202                 $descr =~ s/\s+$//;
2203                 $conf->{description} = $descr;
2204             }
2205             $descr = undef;
2206             $conf = $res->{$section} = {};
2207             next;
2208
2209         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2210             $section = $1;
2211             if (defined($descr)) {
2212                 $descr =~ s/\s+$//;
2213                 $conf->{description} = $descr;
2214             }
2215             $descr = undef;
2216             $conf = $res->{snapshots}->{$section} = {};
2217             next;
2218         }
2219
2220         if ($line =~ m/^\#(.*)\s*$/) {
2221             $descr = '' if !defined($descr);
2222             $descr .= PVE::Tools::decode_text($1) . "\n";
2223             next;
2224         }
2225
2226         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2227             $descr = '' if !defined($descr);
2228             $descr .= PVE::Tools::decode_text($2);
2229         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2230             $conf->{snapstate} = $1;
2231         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2232             my $key = $1;
2233             my $value = $2;
2234             $conf->{$key} = $value;
2235         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2236             my $value = $1;
2237             if ($section eq 'pending') {
2238                 $conf->{delete} = $value; # we parse this later
2239             } else {
2240                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2241             }
2242         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2243             my $key = $1;
2244             my $value = $2;
2245             eval { $value = check_type($key, $value); };
2246             if ($@) {
2247                 warn "vm $vmid - unable to parse value of '$key' - $@";
2248             } else {
2249                 $key = 'ide2' if $key eq 'cdrom';
2250                 my $fmt = $confdesc->{$key}->{format};
2251                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2252                     my $v = parse_drive($key, $value);
2253                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2254                         $v->{file} = $volid;
2255                         $value = print_drive($vmid, $v);
2256                     } else {
2257                         warn "vm $vmid - unable to parse value of '$key'\n";
2258                         next;
2259                     }
2260                 }
2261
2262                 $conf->{$key} = $value;
2263             }
2264         }
2265     }
2266
2267     if (defined($descr)) {
2268         $descr =~ s/\s+$//;
2269         $conf->{description} = $descr;
2270     }
2271     delete $res->{snapstate}; # just to be sure
2272
2273     return $res;
2274 }
2275
2276 sub write_vm_config {
2277     my ($filename, $conf) = @_;
2278
2279     delete $conf->{snapstate}; # just to be sure
2280
2281     if ($conf->{cdrom}) {
2282         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2283         $conf->{ide2} = $conf->{cdrom};
2284         delete $conf->{cdrom};
2285     }
2286
2287     # we do not use 'smp' any longer
2288     if ($conf->{sockets}) {
2289         delete $conf->{smp};
2290     } elsif ($conf->{smp}) {
2291         $conf->{sockets} = $conf->{smp};
2292         delete $conf->{cores};
2293         delete $conf->{smp};
2294     }
2295
2296     my $used_volids = {};
2297
2298     my $cleanup_config = sub {
2299         my ($cref, $pending, $snapname) = @_;
2300
2301         foreach my $key (keys %$cref) {
2302             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2303                 $key eq 'snapstate' || $key eq 'pending';
2304             my $value = $cref->{$key};
2305             if ($key eq 'delete') {
2306                 die "propertry 'delete' is only allowed in [PENDING]\n"
2307                     if !$pending;
2308                 # fixme: check syntax?
2309                 next;
2310             }
2311             eval { $value = check_type($key, $value); };
2312             die "unable to parse value of '$key' - $@" if $@;
2313
2314             $cref->{$key} = $value;
2315
2316             if (!$snapname && is_valid_drivename($key)) {
2317                 my $drive = parse_drive($key, $value);
2318                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2319             }
2320         }
2321     };
2322
2323     &$cleanup_config($conf);
2324
2325     &$cleanup_config($conf->{pending}, 1);
2326
2327     foreach my $snapname (keys %{$conf->{snapshots}}) {
2328         die "internal error" if $snapname eq 'pending';
2329         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2330     }
2331
2332     # remove 'unusedX' settings if we re-add a volume
2333     foreach my $key (keys %$conf) {
2334         my $value = $conf->{$key};
2335         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2336             delete $conf->{$key};
2337         }
2338     }
2339
2340     my $generate_raw_config = sub {
2341         my ($conf, $pending) = @_;
2342
2343         my $raw = '';
2344
2345         # add description as comment to top of file
2346         if (defined(my $descr = $conf->{description})) {
2347             if ($descr) {
2348                 foreach my $cl (split(/\n/, $descr)) {
2349                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2350                 }
2351             } else {
2352                 $raw .= "#\n" if $pending;
2353             }
2354         }
2355
2356         foreach my $key (sort keys %$conf) {
2357             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2358             $raw .= "$key: $conf->{$key}\n";
2359         }
2360         return $raw;
2361     };
2362
2363     my $raw = &$generate_raw_config($conf);
2364
2365     if (scalar(keys %{$conf->{pending}})){
2366         $raw .= "\n[PENDING]\n";
2367         $raw .= &$generate_raw_config($conf->{pending}, 1);
2368     }
2369
2370     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2371         $raw .= "\n[$snapname]\n";
2372         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2373     }
2374
2375     return $raw;
2376 }
2377
2378 sub load_defaults {
2379
2380     my $res = {};
2381
2382     # we use static defaults from our JSON schema configuration
2383     foreach my $key (keys %$confdesc) {
2384         if (defined(my $default = $confdesc->{$key}->{default})) {
2385             $res->{$key} = $default;
2386         }
2387     }
2388
2389     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2390     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2391
2392     return $res;
2393 }
2394
2395 sub config_list {
2396     my $vmlist = PVE::Cluster::get_vmlist();
2397     my $res = {};
2398     return $res if !$vmlist || !$vmlist->{ids};
2399     my $ids = $vmlist->{ids};
2400
2401     foreach my $vmid (keys %$ids) {
2402         my $d = $ids->{$vmid};
2403         next if !$d->{node} || $d->{node} ne $nodename;
2404         next if !$d->{type} || $d->{type} ne 'qemu';
2405         $res->{$vmid}->{exists} = 1;
2406     }
2407     return $res;
2408 }
2409
2410 # test if VM uses local resources (to prevent migration)
2411 sub check_local_resources {
2412     my ($conf, $noerr) = @_;
2413
2414     my $loc_res = 0;
2415
2416     $loc_res = 1 if $conf->{hostusb}; # old syntax
2417     $loc_res = 1 if $conf->{hostpci}; # old syntax
2418
2419     foreach my $k (keys %$conf) {
2420         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2421         # sockets are safe: they will recreated be on the target side post-migrate
2422         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2423         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2424     }
2425
2426     die "VM uses local resources\n" if $loc_res && !$noerr;
2427
2428     return $loc_res;
2429 }
2430
2431 # check if used storages are available on all nodes (use by migrate)
2432 sub check_storage_availability {
2433     my ($storecfg, $conf, $node) = @_;
2434
2435     foreach_drive($conf, sub {
2436         my ($ds, $drive) = @_;
2437
2438         my $volid = $drive->{file};
2439         return if !$volid;
2440
2441         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2442         return if !$sid;
2443
2444         # check if storage is available on both nodes
2445         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2446         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2447    });
2448 }
2449
2450 # list nodes where all VM images are available (used by has_feature API)
2451 sub shared_nodes {
2452     my ($conf, $storecfg) = @_;
2453
2454     my $nodelist = PVE::Cluster::get_nodelist();
2455     my $nodehash = { map { $_ => 1 } @$nodelist };
2456     my $nodename = PVE::INotify::nodename();
2457
2458     foreach_drive($conf, sub {
2459         my ($ds, $drive) = @_;
2460
2461         my $volid = $drive->{file};
2462         return if !$volid;
2463
2464         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2465         if ($storeid) {
2466             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2467             if ($scfg->{disable}) {
2468                 $nodehash = {};
2469             } elsif (my $avail = $scfg->{nodes}) {
2470                 foreach my $node (keys %$nodehash) {
2471                     delete $nodehash->{$node} if !$avail->{$node};
2472                 }
2473             } elsif (!$scfg->{shared}) {
2474                 foreach my $node (keys %$nodehash) {
2475                     delete $nodehash->{$node} if $node ne $nodename
2476                 }
2477             }
2478         }
2479     });
2480
2481     return $nodehash
2482 }
2483
2484 sub check_cmdline {
2485     my ($pidfile, $pid) = @_;
2486
2487     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2488     if (defined($fh)) {
2489         my $line = <$fh>;
2490         $fh->close;
2491         return undef if !$line;
2492         my @param = split(/\0/, $line);
2493
2494         my $cmd = $param[0];
2495         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2496
2497         for (my $i = 0; $i < scalar (@param); $i++) {
2498             my $p = $param[$i];
2499             next if !$p;
2500             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2501                 my $p = $param[$i+1];
2502                 return 1 if $p && ($p eq $pidfile);
2503                 return undef;
2504             }
2505         }
2506     }
2507     return undef;
2508 }
2509
2510 sub check_running {
2511     my ($vmid, $nocheck, $node) = @_;
2512
2513     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2514
2515     die "unable to find configuration file for VM $vmid - no such machine\n"
2516         if !$nocheck && ! -f $filename;
2517
2518     my $pidfile = pidfile_name($vmid);
2519
2520     if (my $fd = IO::File->new("<$pidfile")) {
2521         my $st = stat($fd);
2522         my $line = <$fd>;
2523         close($fd);
2524
2525         my $mtime = $st->mtime;
2526         if ($mtime > time()) {
2527             warn "file '$filename' modified in future\n";
2528         }
2529
2530         if ($line =~ m/^(\d+)$/) {
2531             my $pid = $1;
2532             if (check_cmdline($pidfile, $pid)) {
2533                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2534                     return $pid;
2535                 }
2536             }
2537         }
2538     }
2539
2540     return undef;
2541 }
2542
2543 sub vzlist {
2544
2545     my $vzlist = config_list();
2546
2547     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2548
2549     while (defined(my $de = $fd->read)) {
2550         next if $de !~ m/^(\d+)\.pid$/;
2551         my $vmid = $1;
2552         next if !defined($vzlist->{$vmid});
2553         if (my $pid = check_running($vmid)) {
2554             $vzlist->{$vmid}->{pid} = $pid;
2555         }
2556     }
2557
2558     return $vzlist;
2559 }
2560
2561 sub disksize {
2562     my ($storecfg, $conf) = @_;
2563
2564     my $bootdisk = $conf->{bootdisk};
2565     return undef if !$bootdisk;
2566     return undef if !is_valid_drivename($bootdisk);
2567
2568     return undef if !$conf->{$bootdisk};
2569
2570     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2571     return undef if !defined($drive);
2572
2573     return undef if drive_is_cdrom($drive);
2574
2575     my $volid = $drive->{file};
2576     return undef if !$volid;
2577
2578     return $drive->{size};
2579 }
2580
2581 my $last_proc_pid_stat;
2582
2583 # get VM status information
2584 # This must be fast and should not block ($full == false)
2585 # We only query KVM using QMP if $full == true (this can be slow)
2586 sub vmstatus {
2587     my ($opt_vmid, $full) = @_;
2588
2589     my $res = {};
2590
2591     my $storecfg = PVE::Storage::config();
2592
2593     my $list = vzlist();
2594     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2595
2596     my $cpucount = $cpuinfo->{cpus} || 1;
2597
2598     foreach my $vmid (keys %$list) {
2599         next if $opt_vmid && ($vmid ne $opt_vmid);
2600
2601         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2602         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2603
2604         my $d = {};
2605         $d->{pid} = $list->{$vmid}->{pid};
2606
2607         # fixme: better status?
2608         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2609
2610         my $size = disksize($storecfg, $conf);
2611         if (defined($size)) {
2612             $d->{disk} = 0; # no info available
2613             $d->{maxdisk} = $size;
2614         } else {
2615             $d->{disk} = 0;
2616             $d->{maxdisk} = 0;
2617         }
2618
2619         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2620         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2621         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2622
2623         $d->{name} = $conf->{name} || "VM $vmid";
2624         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2625
2626         if ($conf->{balloon}) {
2627             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2628             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2629         }
2630
2631         $d->{uptime} = 0;
2632         $d->{cpu} = 0;
2633         $d->{mem} = 0;
2634
2635         $d->{netout} = 0;
2636         $d->{netin} = 0;
2637
2638         $d->{diskread} = 0;
2639         $d->{diskwrite} = 0;
2640
2641         $d->{template} = PVE::QemuConfig->is_template($conf);
2642
2643         $res->{$vmid} = $d;
2644     }
2645
2646     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2647     foreach my $dev (keys %$netdev) {
2648         next if $dev !~ m/^tap([1-9]\d*)i/;
2649         my $vmid = $1;
2650         my $d = $res->{$vmid};
2651         next if !$d;
2652
2653         $d->{netout} += $netdev->{$dev}->{receive};
2654         $d->{netin} += $netdev->{$dev}->{transmit};
2655
2656         if ($full) {
2657             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2658             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2659         }
2660
2661     }
2662
2663     my $ctime = gettimeofday;
2664
2665     foreach my $vmid (keys %$list) {
2666
2667         my $d = $res->{$vmid};
2668         my $pid = $d->{pid};
2669         next if !$pid;
2670
2671         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2672         next if !$pstat; # not running
2673
2674         my $used = $pstat->{utime} + $pstat->{stime};
2675
2676         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2677
2678         if ($pstat->{vsize}) {
2679             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2680         }
2681
2682         my $old = $last_proc_pid_stat->{$pid};
2683         if (!$old) {
2684             $last_proc_pid_stat->{$pid} = {
2685                 time => $ctime,
2686                 used => $used,
2687                 cpu => 0,
2688             };
2689             next;
2690         }
2691
2692         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2693
2694         if ($dtime > 1000) {
2695             my $dutime = $used -  $old->{used};
2696
2697             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2698             $last_proc_pid_stat->{$pid} = {
2699                 time => $ctime,
2700                 used => $used,
2701                 cpu => $d->{cpu},
2702             };
2703         } else {
2704             $d->{cpu} = $old->{cpu};
2705         }
2706     }
2707
2708     return $res if !$full;
2709
2710     my $qmpclient = PVE::QMPClient->new();
2711
2712     my $ballooncb = sub {
2713         my ($vmid, $resp) = @_;
2714
2715         my $info = $resp->{'return'};
2716         return if !$info->{max_mem};
2717
2718         my $d = $res->{$vmid};
2719
2720         # use memory assigned to VM
2721         $d->{maxmem} = $info->{max_mem};
2722         $d->{balloon} = $info->{actual};
2723
2724         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2725             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2726             $d->{freemem} = $info->{free_mem};
2727         }
2728
2729         $d->{ballooninfo} = $info;
2730     };
2731
2732     my $blockstatscb = sub {
2733         my ($vmid, $resp) = @_;
2734         my $data = $resp->{'return'} || [];
2735         my $totalrdbytes = 0;
2736         my $totalwrbytes = 0;
2737
2738         for my $blockstat (@$data) {
2739             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2740             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2741
2742             $blockstat->{device} =~ s/drive-//;
2743             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2744         }
2745         $res->{$vmid}->{diskread} = $totalrdbytes;
2746         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2747     };
2748
2749     my $statuscb = sub {
2750         my ($vmid, $resp) = @_;
2751
2752         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2753         # this fails if ballon driver is not loaded, so this must be
2754         # the last commnand (following command are aborted if this fails).
2755         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2756
2757         my $status = 'unknown';
2758         if (!defined($status = $resp->{'return'}->{status})) {
2759             warn "unable to get VM status\n";
2760             return;
2761         }
2762
2763         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2764     };
2765
2766     foreach my $vmid (keys %$list) {
2767         next if $opt_vmid && ($vmid ne $opt_vmid);
2768         next if !$res->{$vmid}->{pid}; # not running
2769         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2770     }
2771
2772     $qmpclient->queue_execute(undef, 2);
2773
2774     foreach my $vmid (keys %$list) {
2775         next if $opt_vmid && ($vmid ne $opt_vmid);
2776         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2777     }
2778
2779     return $res;
2780 }
2781
2782 sub foreach_drive {
2783     my ($conf, $func, @param) = @_;
2784
2785     foreach my $ds (valid_drive_names()) {
2786         next if !defined($conf->{$ds});
2787
2788         my $drive = parse_drive($ds, $conf->{$ds});
2789         next if !$drive;
2790
2791         &$func($ds, $drive, @param);
2792     }
2793 }
2794
2795 sub foreach_volid {
2796     my ($conf, $func, @param) = @_;
2797
2798     my $volhash = {};
2799
2800     my $test_volid = sub {
2801         my ($volid, $is_cdrom, $replicate, $snapname) = @_;
2802
2803         return if !$volid;
2804
2805         $volhash->{$volid}->{cdrom} //= 1;
2806         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2807
2808         $volhash->{$volid}->{replicate} //= 0;
2809         $volhash->{$volid}->{replicate} = 1 if $replicate;
2810
2811         $volhash->{$volid}->{referenced_in_config} //= 0;
2812         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
2813
2814         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
2815             if defined($snapname);
2816     };
2817
2818     foreach_drive($conf, sub {
2819         my ($ds, $drive) = @_;
2820         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, undef);
2821     });
2822
2823     foreach my $snapname (keys %{$conf->{snapshots}}) {
2824         my $snap = $conf->{snapshots}->{$snapname};
2825         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
2826         foreach_drive($snap, sub {
2827             my ($ds, $drive) = @_;
2828             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $snapname);
2829         });
2830     }
2831
2832     foreach my $volid (keys %$volhash) {
2833         &$func($volid, $volhash->{$volid}, @param);
2834     }
2835 }
2836
2837 sub vga_conf_has_spice {
2838     my ($vga) = @_;
2839
2840     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2841
2842     return $1 || 1;
2843 }
2844
2845 sub config_to_command {
2846     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2847
2848     my $cmd = [];
2849     my $globalFlags = [];
2850     my $machineFlags = [];
2851     my $rtcFlags = [];
2852     my $cpuFlags = [];
2853     my $devices = [];
2854     my $pciaddr = '';
2855     my $bridges = {};
2856     my $kvmver = kvm_user_version();
2857     my $vernum = 0; # unknown
2858     my $ostype = $conf->{ostype};
2859     my $winversion = windows_version($ostype);
2860     my $kvm = $conf->{kvm} // 1;
2861
2862     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
2863
2864     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2865         $vernum = $1*1000000+$2*1000;
2866     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2867         $vernum = $1*1000000+$2*1000+$3;
2868     }
2869
2870     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2871
2872     my $have_ovz = -f '/proc/vz/vestat';
2873
2874     my $q35 = machine_type_is_q35($conf);
2875     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2876     my $machine_type = $forcemachine || $conf->{machine};
2877     my $use_old_bios_files = undef;
2878     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2879
2880     my $cpuunits = defined($conf->{cpuunits}) ?
2881             $conf->{cpuunits} : $defaults->{cpuunits};
2882
2883     push @$cmd, '/usr/bin/kvm';
2884
2885     push @$cmd, '-id', $vmid;
2886
2887     my $use_virtio = 0;
2888
2889     my $qmpsocket = qmp_socket($vmid);
2890     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2891     push @$cmd, '-mon', "chardev=qmp,mode=control";
2892
2893
2894     push @$cmd, '-pidfile' , pidfile_name($vmid);
2895
2896     push @$cmd, '-daemonize';
2897
2898     if ($conf->{smbios1}) {
2899         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2900     }
2901
2902     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2903         die "uefi base image not found\n" if ! -f $OVMF_CODE;
2904
2905         my $path;
2906         my $format = 'raw';
2907         if (my $efidisk = $conf->{efidisk0}) {
2908             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
2909             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2910             if ($storeid) {
2911                 $path = PVE::Storage::path($storecfg, $d->{file});
2912             } else {
2913                 $path = $d->{file};
2914             }
2915             $format = $d->{format} if $d->{format};
2916         } else {
2917             warn "no efidisk configured! Using temporary efivars disk.\n";
2918             $path = "/tmp/$vmid-ovmf.fd";
2919             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
2920         }
2921
2922         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
2923         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
2924     }
2925
2926
2927     # add usb controllers
2928     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2929     push @$devices, @usbcontrollers if @usbcontrollers;
2930     my $vga = $conf->{vga};
2931
2932     my $qxlnum = vga_conf_has_spice($vga);
2933     $vga = 'qxl' if $qxlnum;
2934
2935     if (!$vga) {
2936         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
2937             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
2938         } else {
2939             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
2940         }
2941     }
2942
2943     # enable absolute mouse coordinates (needed by vnc)
2944     my $tablet;
2945     if (defined($conf->{tablet})) {
2946         $tablet = $conf->{tablet};
2947     } else {
2948         $tablet = $defaults->{tablet};
2949         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2950         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2951     }
2952
2953     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2954
2955     my $kvm_off = 0;
2956     my $gpu_passthrough;
2957
2958     # host pci devices
2959     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2960         my $d = parse_hostpci($conf->{"hostpci$i"});
2961         next if !$d;
2962
2963         my $pcie = $d->{pcie};
2964         if($pcie){
2965             die "q35 machine model is not enabled" if !$q35;
2966             $pciaddr = print_pcie_addr("hostpci$i");
2967         }else{
2968             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2969         }
2970
2971         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2972         my $romfile = $d->{romfile};
2973
2974         my $xvga = '';
2975         if ($d->{'x-vga'}) {
2976             $xvga = ',x-vga=on';
2977             $kvm_off = 1;
2978             $vga = 'none';
2979             $gpu_passthrough = 1;
2980
2981             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2982                 $xvga = "";
2983             }
2984         }
2985         my $pcidevices = $d->{pciid};
2986         my $multifunction = 1 if @$pcidevices > 1;
2987
2988         my $j=0;
2989         foreach my $pcidevice (@$pcidevices) {
2990
2991             my $id = "hostpci$i";
2992             $id .= ".$j" if $multifunction;
2993             my $addr = $pciaddr;
2994             $addr .= ".$j" if $multifunction;
2995             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2996
2997             if($j == 0){
2998                 $devicestr .= "$rombar$xvga";
2999                 $devicestr .= ",multifunction=on" if $multifunction;
3000                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3001             }
3002
3003             push @$devices, '-device', $devicestr;
3004             $j++;
3005         }
3006     }
3007
3008     # usb devices
3009     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3010     push @$devices, @usbdevices if @usbdevices;
3011     # serial devices
3012     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3013         if (my $path = $conf->{"serial$i"}) {
3014             if ($path eq 'socket') {
3015                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3016                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3017                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3018             } else {
3019                 die "no such serial device\n" if ! -c $path;
3020                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3021                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3022             }
3023         }
3024     }
3025
3026     # parallel devices
3027     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3028         if (my $path = $conf->{"parallel$i"}) {
3029             die "no such parallel device\n" if ! -c $path;
3030             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3031             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3032             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3033         }
3034     }
3035
3036     my $vmname = $conf->{name} || "vm$vmid";
3037
3038     push @$cmd, '-name', $vmname;
3039
3040     my $sockets = 1;
3041     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3042     $sockets = $conf->{sockets} if  $conf->{sockets};
3043
3044     my $cores = $conf->{cores} || 1;
3045
3046     my $maxcpus = $sockets * $cores;
3047
3048     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3049
3050     my $allowed_vcpus = $cpuinfo->{cpus};
3051
3052     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3053         if ($allowed_vcpus < $maxcpus);
3054
3055     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3056
3057         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3058         for (my $i = 2; $i <= $vcpus; $i++)  {
3059             my $cpustr = print_cpu_device($conf,$i);
3060             push @$cmd, '-device', $cpustr;
3061         }
3062
3063     } else {
3064
3065         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3066     }
3067     push @$cmd, '-nodefaults';
3068
3069     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3070
3071     my $bootindex_hash = {};
3072     my $i = 1;
3073     foreach my $o (split(//, $bootorder)) {
3074         $bootindex_hash->{$o} = $i*100;
3075         $i++;
3076     }
3077
3078     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3079
3080     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3081
3082     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3083
3084     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3085
3086     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3087         my $socket = vnc_socket($vmid);
3088         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3089     } else {
3090         push @$cmd, '-nographic';
3091     }
3092
3093     # time drift fix
3094     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3095
3096     my $useLocaltime = $conf->{localtime};
3097
3098     if ($winversion >= 5) { # windows
3099         $useLocaltime = 1 if !defined($conf->{localtime});
3100
3101         # use time drift fix when acpi is enabled
3102         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3103             $tdf = 1 if !defined($conf->{tdf});
3104         }
3105     }
3106
3107     if ($winversion >= 6) {
3108         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3109         push @$cmd, '-no-hpet';
3110     }
3111
3112     push @$rtcFlags, 'driftfix=slew' if $tdf;
3113
3114     if (!$kvm) {
3115         push @$machineFlags, 'accel=tcg';
3116     }
3117
3118     if ($machine_type) {
3119         push @$machineFlags, "type=${machine_type}";
3120     }
3121
3122     if ($conf->{startdate}) {
3123         push @$rtcFlags, "base=$conf->{startdate}";
3124     } elsif ($useLocaltime) {
3125         push @$rtcFlags, 'base=localtime';
3126     }
3127
3128     my $cpu = $kvm ? "kvm64" : "qemu64";
3129     if (my $cputype = $conf->{cpu}) {
3130         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3131             or die "Cannot parse cpu description: $cputype\n";
3132         $cpu = $cpuconf->{cputype};
3133         $kvm_off = 1 if $cpuconf->{hidden};
3134     }
3135
3136     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3137
3138     push @$cpuFlags , '-x2apic'
3139         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3140
3141     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3142
3143     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3144
3145     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3146
3147         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3148         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3149     }
3150
3151     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3152
3153     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3154
3155     push @$cpuFlags, 'kvm=off' if $kvm_off;
3156
3157     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3158         die "internal error"; # should not happen
3159
3160     push @$cpuFlags, "vendor=${cpu_vendor}"
3161         if $cpu_vendor ne 'default';
3162
3163     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3164
3165     push @$cmd, '-cpu', $cpu;
3166
3167     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3168
3169     push @$cmd, '-S' if $conf->{freeze};
3170
3171     # set keyboard layout
3172     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3173     push @$cmd, '-k', $kb if $kb;
3174
3175     # enable sound
3176     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3177     #push @$cmd, '-soundhw', 'es1370';
3178     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3179
3180     if($conf->{agent}) {
3181         my $qgasocket = qmp_socket($vmid, 1);
3182         my $pciaddr = print_pci_addr("qga0", $bridges);
3183         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3184         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3185         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3186     }
3187
3188     my $spice_port;
3189
3190     if ($qxlnum) {
3191         if ($qxlnum > 1) {
3192             if ($winversion){
3193                 for(my $i = 1; $i < $qxlnum; $i++){
3194                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3195                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3196                 }
3197             } else {
3198                 # assume other OS works like Linux
3199                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3200                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3201             }
3202         }
3203
3204         my $pciaddr = print_pci_addr("spice", $bridges);
3205
3206         my $nodename = PVE::INotify::nodename();
3207         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3208         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3209         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3210         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3211         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3212
3213         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3214
3215         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3216         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3217         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3218     }
3219
3220     # enable balloon by default, unless explicitly disabled
3221     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3222         $pciaddr = print_pci_addr("balloon0", $bridges);
3223         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3224     }
3225
3226     if ($conf->{watchdog}) {
3227         my $wdopts = parse_watchdog($conf->{watchdog});
3228         $pciaddr = print_pci_addr("watchdog", $bridges);
3229         my $watchdog = $wdopts->{model} || 'i6300esb';
3230         push @$devices, '-device', "$watchdog$pciaddr";
3231         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3232     }
3233
3234     my $vollist = [];
3235     my $scsicontroller = {};
3236     my $ahcicontroller = {};
3237     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3238
3239     # Add iscsi initiator name if available
3240     if (my $initiator = get_initiator_name()) {
3241         push @$devices, '-iscsi', "initiator-name=$initiator";
3242     }
3243
3244     foreach_drive($conf, sub {
3245         my ($ds, $drive) = @_;
3246
3247         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3248             push @$vollist, $drive->{file};
3249         }
3250
3251         # ignore efidisk here, already added in bios/fw handling code above
3252         return if $drive->{interface} eq 'efidisk';
3253
3254         $use_virtio = 1 if $ds =~ m/^virtio/;
3255
3256         if (drive_is_cdrom ($drive)) {
3257             if ($bootindex_hash->{d}) {
3258                 $drive->{bootindex} = $bootindex_hash->{d};
3259                 $bootindex_hash->{d} += 1;
3260             }
3261         } else {
3262             if ($bootindex_hash->{c}) {
3263                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3264                 $bootindex_hash->{c} += 1;
3265             }
3266         }
3267
3268         if($drive->{interface} eq 'virtio'){
3269            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3270         }
3271
3272         if ($drive->{interface} eq 'scsi') {
3273
3274             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3275
3276             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3277             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3278
3279             my $iothread = '';
3280             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3281                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3282                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3283             } elsif ($drive->{iothread}) {
3284                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3285             }
3286
3287             my $queues = '';
3288             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3289                 $queues = ",num_queues=$drive->{queues}";
3290             }
3291
3292             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3293             $scsicontroller->{$controller}=1;
3294         }
3295
3296         if ($drive->{interface} eq 'sata') {
3297            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3298            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3299            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3300            $ahcicontroller->{$controller}=1;
3301         }
3302
3303         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3304         push @$devices, '-drive',$drive_cmd;
3305         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3306     });
3307
3308     for (my $i = 0; $i < $MAX_NETS; $i++) {
3309          next if !$conf->{"net$i"};
3310          my $d = parse_net($conf->{"net$i"});
3311          next if !$d;
3312
3313          $use_virtio = 1 if $d->{model} eq 'virtio';
3314
3315          if ($bootindex_hash->{n}) {
3316             $d->{bootindex} = $bootindex_hash->{n};
3317             $bootindex_hash->{n} += 1;
3318          }
3319
3320          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3321          push @$devices, '-netdev', $netdevfull;
3322
3323          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3324          push @$devices, '-device', $netdevicefull;
3325     }
3326
3327     if (!$q35) {
3328         # add pci bridges
3329         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3330            $bridges->{1} = 1;
3331            $bridges->{2} = 1;
3332         }
3333
3334         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3335
3336         while (my ($k, $v) = each %$bridges) {
3337             $pciaddr = print_pci_addr("pci.$k");
3338             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3339         }
3340     }
3341
3342     # add custom args
3343     if ($conf->{args}) {
3344         my $aa = PVE::Tools::split_args($conf->{args});
3345         push @$cmd, @$aa;
3346     }
3347
3348     push @$cmd, @$devices;
3349     push @$cmd, '-rtc', join(',', @$rtcFlags)
3350         if scalar(@$rtcFlags);
3351     push @$cmd, '-machine', join(',', @$machineFlags)
3352         if scalar(@$machineFlags);
3353     push @$cmd, '-global', join(',', @$globalFlags)
3354         if scalar(@$globalFlags);
3355
3356     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3357 }
3358
3359 sub vnc_socket {
3360     my ($vmid) = @_;
3361     return "${var_run_tmpdir}/$vmid.vnc";
3362 }
3363
3364 sub spice_port {
3365     my ($vmid) = @_;
3366
3367     my $res = vm_mon_cmd($vmid, 'query-spice');
3368
3369     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3370 }
3371
3372 sub qmp_socket {
3373     my ($vmid, $qga) = @_;
3374     my $sockettype = $qga ? 'qga' : 'qmp';
3375     return "${var_run_tmpdir}/$vmid.$sockettype";
3376 }
3377
3378 sub pidfile_name {
3379     my ($vmid) = @_;
3380     return "${var_run_tmpdir}/$vmid.pid";
3381 }
3382
3383 sub vm_devices_list {
3384     my ($vmid) = @_;
3385
3386     my $res = vm_mon_cmd($vmid, 'query-pci');
3387     my $devices = {};
3388     foreach my $pcibus (@$res) {
3389         foreach my $device (@{$pcibus->{devices}}) {
3390             next if !$device->{'qdev_id'};
3391             if ($device->{'pci_bridge'}) {
3392                 $devices->{$device->{'qdev_id'}} = 1;
3393                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3394                     next if !$bridge_device->{'qdev_id'};
3395                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3396                     $devices->{$device->{'qdev_id'}}++;
3397                 }
3398             } else {
3399                 $devices->{$device->{'qdev_id'}} = 1;
3400             }
3401         }
3402     }
3403
3404     my $resblock = vm_mon_cmd($vmid, 'query-block');
3405     foreach my $block (@$resblock) {
3406         if($block->{device} =~ m/^drive-(\S+)/){
3407                 $devices->{$1} = 1;
3408         }
3409     }
3410
3411     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3412     foreach my $mice (@$resmice) {
3413         if ($mice->{name} eq 'QEMU HID Tablet') {
3414             $devices->{tablet} = 1;
3415             last;
3416         }
3417     }
3418
3419     # for usb devices there is no query-usb
3420     # but we can iterate over the entries in
3421     # qom-list path=/machine/peripheral
3422     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3423     foreach my $per (@$resperipheral) {
3424         if ($per->{name} =~ m/^usb\d+$/) {
3425             $devices->{$per->{name}} = 1;
3426         }
3427     }
3428
3429     return $devices;
3430 }
3431
3432 sub vm_deviceplug {
3433     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3434
3435     my $q35 = machine_type_is_q35($conf);
3436
3437     my $devices_list = vm_devices_list($vmid);
3438     return 1 if defined($devices_list->{$deviceid});
3439
3440     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3441
3442     if ($deviceid eq 'tablet') {
3443
3444         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3445
3446     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3447
3448         die "usb hotplug currently not reliable\n";
3449         # since we can't reliably hot unplug all added usb devices
3450         # and usb passthrough disables live migration
3451         # we disable usb hotplugging for now
3452         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3453
3454     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3455
3456         qemu_iothread_add($vmid, $deviceid, $device);
3457
3458         qemu_driveadd($storecfg, $vmid, $device);
3459         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3460
3461         qemu_deviceadd($vmid, $devicefull);
3462         eval { qemu_deviceaddverify($vmid, $deviceid); };
3463         if (my $err = $@) {
3464             eval { qemu_drivedel($vmid, $deviceid); };
3465             warn $@ if $@;
3466             die $err;
3467         }
3468
3469     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3470
3471
3472         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3473         my $pciaddr = print_pci_addr($deviceid);
3474         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3475
3476         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3477
3478         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3479             qemu_iothread_add($vmid, $deviceid, $device);
3480             $devicefull .= ",iothread=iothread-$deviceid";
3481         }
3482
3483         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3484             $devicefull .= ",num_queues=$device->{queues}";
3485         }
3486
3487         qemu_deviceadd($vmid, $devicefull);
3488         qemu_deviceaddverify($vmid, $deviceid);
3489
3490     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3491
3492         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3493         qemu_driveadd($storecfg, $vmid, $device);
3494
3495         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3496         eval { qemu_deviceadd($vmid, $devicefull); };
3497         if (my $err = $@) {
3498             eval { qemu_drivedel($vmid, $deviceid); };
3499             warn $@ if $@;
3500             die $err;
3501         }
3502
3503     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3504
3505         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3506
3507         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3508         my $use_old_bios_files = undef;
3509         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3510
3511         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3512         qemu_deviceadd($vmid, $netdevicefull);
3513         eval { qemu_deviceaddverify($vmid, $deviceid); };
3514         if (my $err = $@) {
3515             eval { qemu_netdevdel($vmid, $deviceid); };
3516             warn $@ if $@;
3517             die $err;
3518         }
3519
3520     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3521
3522         my $bridgeid = $2;
3523         my $pciaddr = print_pci_addr($deviceid);
3524         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3525
3526         qemu_deviceadd($vmid, $devicefull);
3527         qemu_deviceaddverify($vmid, $deviceid);
3528
3529     } else {
3530         die "can't hotplug device '$deviceid'\n";
3531     }
3532
3533     return 1;
3534 }
3535
3536 # fixme: this should raise exceptions on error!
3537 sub vm_deviceunplug {
3538     my ($vmid, $conf, $deviceid) = @_;
3539
3540     my $devices_list = vm_devices_list($vmid);
3541     return 1 if !defined($devices_list->{$deviceid});
3542
3543     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3544
3545     if ($deviceid eq 'tablet') {
3546
3547         qemu_devicedel($vmid, $deviceid);
3548
3549     } elsif ($deviceid =~ m/^usb\d+$/) {
3550
3551         die "usb hotplug currently not reliable\n";
3552         # when unplugging usb devices this way,
3553         # there may be remaining usb controllers/hubs
3554         # so we disable it for now
3555         qemu_devicedel($vmid, $deviceid);
3556         qemu_devicedelverify($vmid, $deviceid);
3557
3558     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3559
3560         qemu_devicedel($vmid, $deviceid);
3561         qemu_devicedelverify($vmid, $deviceid);
3562         qemu_drivedel($vmid, $deviceid);
3563         qemu_iothread_del($conf, $vmid, $deviceid);
3564
3565     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3566
3567         qemu_devicedel($vmid, $deviceid);
3568         qemu_devicedelverify($vmid, $deviceid);
3569         qemu_iothread_del($conf, $vmid, $deviceid);
3570
3571     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3572
3573         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3574         my $device = parse_drive($deviceid, $conf->{$deviceid});
3575         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3576
3577         qemu_devicedel($vmid, $deviceid);
3578         qemu_drivedel($vmid, $deviceid);
3579         qemu_deletescsihw($conf, $vmid, $deviceid);
3580
3581     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3582
3583         qemu_devicedel($vmid, $deviceid);
3584         qemu_devicedelverify($vmid, $deviceid);
3585         qemu_netdevdel($vmid, $deviceid);
3586
3587     } else {
3588         die "can't unplug device '$deviceid'\n";
3589     }
3590
3591     return 1;
3592 }
3593
3594 sub qemu_deviceadd {
3595     my ($vmid, $devicefull) = @_;
3596
3597     $devicefull = "driver=".$devicefull;
3598     my %options =  split(/[=,]/, $devicefull);
3599
3600     vm_mon_cmd($vmid, "device_add" , %options);
3601 }
3602
3603 sub qemu_devicedel {
3604     my ($vmid, $deviceid) = @_;
3605
3606     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3607 }
3608
3609 sub qemu_iothread_add {
3610     my($vmid, $deviceid, $device) = @_;
3611
3612     if ($device->{iothread}) {
3613         my $iothreads = vm_iothreads_list($vmid);
3614         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3615     }
3616 }
3617
3618 sub qemu_iothread_del {
3619     my($conf, $vmid, $deviceid) = @_;
3620
3621     my $device = parse_drive($deviceid, $conf->{$deviceid});
3622     if ($device->{iothread}) {
3623         my $iothreads = vm_iothreads_list($vmid);
3624         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3625     }
3626 }
3627
3628 sub qemu_objectadd {
3629     my($vmid, $objectid, $qomtype) = @_;
3630
3631     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3632
3633     return 1;
3634 }
3635
3636 sub qemu_objectdel {
3637     my($vmid, $objectid) = @_;
3638
3639     vm_mon_cmd($vmid, "object-del", id => $objectid);
3640
3641     return 1;
3642 }
3643
3644 sub qemu_driveadd {
3645     my ($storecfg, $vmid, $device) = @_;
3646
3647     my $drive = print_drive_full($storecfg, $vmid, $device);
3648     $drive =~ s/\\/\\\\/g;
3649     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3650
3651     # If the command succeeds qemu prints: "OK"
3652     return 1 if $ret =~ m/OK/s;
3653
3654     die "adding drive failed: $ret\n";
3655 }
3656
3657 sub qemu_drivedel {
3658     my($vmid, $deviceid) = @_;
3659
3660     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3661     $ret =~ s/^\s+//;
3662
3663     return 1 if $ret eq "";
3664
3665     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3666     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3667
3668     die "deleting drive $deviceid failed : $ret\n";
3669 }
3670
3671 sub qemu_deviceaddverify {
3672     my ($vmid, $deviceid) = @_;
3673
3674     for (my $i = 0; $i <= 5; $i++) {
3675          my $devices_list = vm_devices_list($vmid);
3676          return 1 if defined($devices_list->{$deviceid});
3677          sleep 1;
3678     }
3679
3680     die "error on hotplug device '$deviceid'\n";
3681 }
3682
3683
3684 sub qemu_devicedelverify {
3685     my ($vmid, $deviceid) = @_;
3686
3687     # need to verify that the device is correctly removed as device_del
3688     # is async and empty return is not reliable
3689
3690     for (my $i = 0; $i <= 5; $i++) {
3691          my $devices_list = vm_devices_list($vmid);
3692          return 1 if !defined($devices_list->{$deviceid});
3693          sleep 1;
3694     }
3695
3696     die "error on hot-unplugging device '$deviceid'\n";
3697 }
3698
3699 sub qemu_findorcreatescsihw {
3700     my ($storecfg, $conf, $vmid, $device) = @_;
3701
3702     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3703
3704     my $scsihwid="$controller_prefix$controller";
3705     my $devices_list = vm_devices_list($vmid);
3706
3707     if(!defined($devices_list->{$scsihwid})) {
3708         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3709     }
3710
3711     return 1;
3712 }
3713
3714 sub qemu_deletescsihw {
3715     my ($conf, $vmid, $opt) = @_;
3716
3717     my $device = parse_drive($opt, $conf->{$opt});
3718
3719     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3720         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3721         return 1;
3722     }
3723
3724     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3725
3726     my $devices_list = vm_devices_list($vmid);
3727     foreach my $opt (keys %{$devices_list}) {
3728         if (PVE::QemuServer::is_valid_drivename($opt)) {
3729             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3730             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3731                 return 1;
3732             }
3733         }
3734     }
3735
3736     my $scsihwid="scsihw$controller";
3737
3738     vm_deviceunplug($vmid, $conf, $scsihwid);
3739
3740     return 1;
3741 }
3742
3743 sub qemu_add_pci_bridge {
3744     my ($storecfg, $conf, $vmid, $device) = @_;
3745
3746     my $bridges = {};
3747
3748     my $bridgeid;
3749
3750     print_pci_addr($device, $bridges);
3751
3752     while (my ($k, $v) = each %$bridges) {
3753         $bridgeid = $k;
3754     }
3755     return 1 if !defined($bridgeid) || $bridgeid < 1;
3756
3757     my $bridge = "pci.$bridgeid";
3758     my $devices_list = vm_devices_list($vmid);
3759
3760     if (!defined($devices_list->{$bridge})) {
3761         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3762     }
3763
3764     return 1;
3765 }
3766
3767 sub qemu_set_link_status {
3768     my ($vmid, $device, $up) = @_;
3769
3770     vm_mon_cmd($vmid, "set_link", name => $device,
3771                up => $up ? JSON::true : JSON::false);
3772 }
3773
3774 sub qemu_netdevadd {
3775     my ($vmid, $conf, $device, $deviceid) = @_;
3776
3777     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3778     my %options =  split(/[=,]/, $netdev);
3779
3780     vm_mon_cmd($vmid, "netdev_add",  %options);
3781     return 1;
3782 }
3783
3784 sub qemu_netdevdel {
3785     my ($vmid, $deviceid) = @_;
3786
3787     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3788 }
3789
3790 sub qemu_usb_hotplug {
3791     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3792
3793     return if !$device;
3794
3795     # remove the old one first
3796     vm_deviceunplug($vmid, $conf, $deviceid);
3797
3798     # check if xhci controller is necessary and available
3799     if ($device->{usb3}) {
3800
3801         my $devicelist = vm_devices_list($vmid);
3802
3803         if (!$devicelist->{xhci}) {
3804             my $pciaddr = print_pci_addr("xhci");
3805             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3806         }
3807     }
3808     my $d = parse_usb_device($device->{host});
3809     $d->{usb3} = $device->{usb3};
3810
3811     # add the new one
3812     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3813 }
3814
3815 sub qemu_cpu_hotplug {
3816     my ($vmid, $conf, $vcpus) = @_;
3817
3818     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3819
3820     my $sockets = 1;
3821     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3822     $sockets = $conf->{sockets} if  $conf->{sockets};
3823     my $cores = $conf->{cores} || 1;
3824     my $maxcpus = $sockets * $cores;
3825
3826     $vcpus = $maxcpus if !$vcpus;
3827
3828     die "you can't add more vcpus than maxcpus\n"
3829         if $vcpus > $maxcpus;
3830
3831     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3832
3833     if ($vcpus < $currentvcpus) {
3834
3835         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3836
3837             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3838                 qemu_devicedel($vmid, "cpu$i");
3839                 my $retry = 0;
3840                 my $currentrunningvcpus = undef;
3841                 while (1) {
3842                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3843                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3844                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3845                     $retry++;
3846                     sleep 1;
3847                 }
3848                 #update conf after each succesfull cpu unplug
3849                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3850                 PVE::QemuConfig->write_config($vmid, $conf);
3851             }
3852         } else {
3853             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3854         }
3855
3856         return;
3857     }
3858
3859     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3860     die "vcpus in running vm does not match its configuration\n"
3861         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3862
3863     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3864
3865         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3866             my $cpustr = print_cpu_device($conf, $i);
3867             qemu_deviceadd($vmid, $cpustr);
3868
3869             my $retry = 0;
3870             my $currentrunningvcpus = undef;
3871             while (1) {
3872                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3873                 last if scalar(@{$currentrunningvcpus}) == $i;
3874                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3875                 sleep 1;
3876                 $retry++;
3877             }
3878             #update conf after each succesfull cpu hotplug
3879             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3880             PVE::QemuConfig->write_config($vmid, $conf);
3881         }
3882     } else {
3883
3884         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3885             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3886         }
3887     }
3888 }
3889
3890 sub qemu_block_set_io_throttle {
3891     my ($vmid, $deviceid,
3892         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3893         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3894         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3895         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3896
3897     return if !check_running($vmid) ;
3898
3899     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3900         bps => int($bps),
3901         bps_rd => int($bps_rd),
3902         bps_wr => int($bps_wr),
3903         iops => int($iops),
3904         iops_rd => int($iops_rd),
3905         iops_wr => int($iops_wr),
3906         bps_max => int($bps_max),
3907         bps_rd_max => int($bps_rd_max),
3908         bps_wr_max => int($bps_wr_max),
3909         iops_max => int($iops_max),
3910         iops_rd_max => int($iops_rd_max),
3911         iops_wr_max => int($iops_wr_max),
3912         bps_max_length => int($bps_max_length),
3913         bps_rd_max_length => int($bps_rd_max_length),
3914         bps_wr_max_length => int($bps_wr_max_length),
3915         iops_max_length => int($iops_max_length),
3916         iops_rd_max_length => int($iops_rd_max_length),
3917         iops_wr_max_length => int($iops_wr_max_length),
3918     );
3919
3920 }
3921
3922 # old code, only used to shutdown old VM after update
3923 sub __read_avail {
3924     my ($fh, $timeout) = @_;
3925
3926     my $sel = new IO::Select;
3927     $sel->add($fh);
3928
3929     my $res = '';
3930     my $buf;
3931
3932     my @ready;
3933     while (scalar (@ready = $sel->can_read($timeout))) {
3934         my $count;
3935         if ($count = $fh->sysread($buf, 8192)) {
3936             if ($buf =~ /^(.*)\(qemu\) $/s) {
3937                 $res .= $1;
3938                 last;
3939             } else {
3940                 $res .= $buf;
3941             }
3942         } else {
3943             if (!defined($count)) {
3944                 die "$!\n";
3945             }
3946             last;
3947         }
3948     }
3949
3950     die "monitor read timeout\n" if !scalar(@ready);
3951
3952     return $res;
3953 }
3954
3955 # old code, only used to shutdown old VM after update
3956 sub vm_monitor_command {
3957     my ($vmid, $cmdstr, $nocheck) = @_;
3958
3959     my $res;
3960
3961     eval {
3962         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3963
3964         my $sname = "${var_run_tmpdir}/$vmid.mon";
3965
3966         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3967             die "unable to connect to VM $vmid socket - $!\n";
3968
3969         my $timeout = 3;
3970
3971         # hack: migrate sometime blocks the monitor (when migrate_downtime
3972         # is set)
3973         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3974             $timeout = 60*60; # 1 hour
3975         }
3976
3977         # read banner;
3978         my $data = __read_avail($sock, $timeout);
3979
3980         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3981             die "got unexpected qemu monitor banner\n";
3982         }
3983
3984         my $sel = new IO::Select;
3985         $sel->add($sock);
3986
3987         if (!scalar(my @ready = $sel->can_write($timeout))) {
3988             die "monitor write error - timeout";
3989         }
3990
3991         my $fullcmd = "$cmdstr\r";
3992
3993         # syslog('info', "VM $vmid monitor command: $cmdstr");
3994
3995         my $b;
3996         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3997             die "monitor write error - $!";
3998         }
3999
4000         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4001
4002         $timeout = 20;
4003
4004         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4005             $timeout = 60*60; # 1 hour
4006         } elsif ($cmdstr =~ m/^(eject|change)/) {
4007             $timeout = 60; # note: cdrom mount command is slow
4008         }
4009         if ($res = __read_avail($sock, $timeout)) {
4010
4011             my @lines = split("\r?\n", $res);
4012
4013             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4014
4015             $res = join("\n", @lines);
4016             $res .= "\n";
4017         }
4018     };
4019
4020     my $err = $@;
4021
4022     if ($err) {
4023         syslog("err", "VM $vmid monitor command failed - $err");
4024         die $err;
4025     }
4026
4027     return $res;
4028 }
4029
4030 sub qemu_block_resize {
4031     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4032
4033     my $running = check_running($vmid);
4034
4035     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4036
4037     return if !$running;
4038
4039     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4040
4041 }
4042
4043 sub qemu_volume_snapshot {
4044     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4045
4046     my $running = check_running($vmid);
4047
4048     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4049         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4050     } else {
4051         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4052     }
4053 }
4054
4055 sub qemu_volume_snapshot_delete {
4056     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4057
4058     my $running = check_running($vmid);
4059
4060     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4061         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4062     } else {
4063         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4064     }
4065 }
4066
4067 sub set_migration_caps {
4068     my ($vmid) = @_;
4069
4070     my $cap_ref = [];
4071
4072     my $enabled_cap = {
4073         "auto-converge" => 1,
4074         "xbzrle" => 1,
4075         "x-rdma-pin-all" => 0,
4076         "zero-blocks" => 0,
4077         "compress" => 0
4078     };
4079
4080     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4081
4082     for my $supported_capability (@$supported_capabilities) {
4083         push @$cap_ref, {
4084             capability => $supported_capability->{capability},
4085             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4086         };
4087     }
4088
4089     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4090 }
4091
4092 my $fast_plug_option = {
4093     'lock' => 1,
4094     'name' => 1,
4095     'onboot' => 1,
4096     'shares' => 1,
4097     'startup' => 1,
4098     'description' => 1,
4099     'protection' => 1,
4100     'vmstatestorage' => 1,
4101 };
4102
4103 # hotplug changes in [PENDING]
4104 # $selection hash can be used to only apply specified options, for
4105 # example: { cores => 1 } (only apply changed 'cores')
4106 # $errors ref is used to return error messages
4107 sub vmconfig_hotplug_pending {
4108     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4109
4110     my $defaults = load_defaults();
4111
4112     # commit values which do not have any impact on running VM first
4113     # Note: those option cannot raise errors, we we do not care about
4114     # $selection and always apply them.
4115
4116     my $add_error = sub {
4117         my ($opt, $msg) = @_;
4118         $errors->{$opt} = "hotplug problem - $msg";
4119     };
4120
4121     my $changes = 0;
4122     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4123         if ($fast_plug_option->{$opt}) {
4124             $conf->{$opt} = $conf->{pending}->{$opt};
4125             delete $conf->{pending}->{$opt};
4126             $changes = 1;
4127         }
4128     }
4129
4130     if ($changes) {
4131         PVE::QemuConfig->write_config($vmid, $conf);
4132         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4133     }
4134
4135     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4136
4137     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4138     while (my ($opt, $force) = each %$pending_delete_hash) {
4139         next if $selection && !$selection->{$opt};
4140         eval {
4141             if ($opt eq 'hotplug') {
4142                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4143             } elsif ($opt eq 'tablet') {
4144                 die "skip\n" if !$hotplug_features->{usb};
4145                 if ($defaults->{tablet}) {
4146                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4147                 } else {
4148                     vm_deviceunplug($vmid, $conf, $opt);
4149                 }
4150             } elsif ($opt =~ m/^usb\d+/) {
4151                 die "skip\n";
4152                 # since we cannot reliably hot unplug usb devices
4153                 # we are disabling it
4154                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4155                 vm_deviceunplug($vmid, $conf, $opt);
4156             } elsif ($opt eq 'vcpus') {
4157                 die "skip\n" if !$hotplug_features->{cpu};
4158                 qemu_cpu_hotplug($vmid, $conf, undef);
4159             } elsif ($opt eq 'balloon') {
4160                 # enable balloon device is not hotpluggable
4161                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4162             } elsif ($fast_plug_option->{$opt}) {
4163                 # do nothing
4164             } elsif ($opt =~ m/^net(\d+)$/) {
4165                 die "skip\n" if !$hotplug_features->{network};
4166                 vm_deviceunplug($vmid, $conf, $opt);
4167             } elsif (is_valid_drivename($opt)) {
4168                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4169                 vm_deviceunplug($vmid, $conf, $opt);
4170                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4171             } elsif ($opt =~ m/^memory$/) {
4172                 die "skip\n" if !$hotplug_features->{memory};
4173                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4174             } elsif ($opt eq 'cpuunits') {
4175                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4176             } elsif ($opt eq 'cpulimit') {
4177                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4178             } else {
4179                 die "skip\n";
4180             }
4181         };
4182         if (my $err = $@) {
4183             &$add_error($opt, $err) if $err ne "skip\n";
4184         } else {
4185             # save new config if hotplug was successful
4186             delete $conf->{$opt};
4187             vmconfig_undelete_pending_option($conf, $opt);
4188             PVE::QemuConfig->write_config($vmid, $conf);
4189             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4190         }
4191     }
4192
4193     foreach my $opt (keys %{$conf->{pending}}) {
4194         next if $selection && !$selection->{$opt};
4195         my $value = $conf->{pending}->{$opt};
4196         eval {
4197             if ($opt eq 'hotplug') {
4198                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4199             } elsif ($opt eq 'tablet') {
4200                 die "skip\n" if !$hotplug_features->{usb};
4201                 if ($value == 1) {
4202                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4203                 } elsif ($value == 0) {
4204                     vm_deviceunplug($vmid, $conf, $opt);
4205                 }
4206             } elsif ($opt =~ m/^usb\d+$/) {
4207                 die "skip\n";
4208                 # since we cannot reliably hot unplug usb devices
4209                 # we are disabling it
4210                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4211                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4212                 die "skip\n" if !$d;
4213                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4214             } elsif ($opt eq 'vcpus') {
4215                 die "skip\n" if !$hotplug_features->{cpu};
4216                 qemu_cpu_hotplug($vmid, $conf, $value);
4217             } elsif ($opt eq 'balloon') {
4218                 # enable/disable balloning device is not hotpluggable
4219                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4220                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4221                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4222
4223                 # allow manual ballooning if shares is set to zero
4224                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4225                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4226                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4227                 }
4228             } elsif ($opt =~ m/^net(\d+)$/) {
4229                 # some changes can be done without hotplug
4230                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4231                                     $vmid, $opt, $value);
4232             } elsif (is_valid_drivename($opt)) {
4233                 # some changes can be done without hotplug
4234                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4235                                      $vmid, $opt, $value, 1);
4236             } elsif ($opt =~ m/^memory$/) { #dimms
4237                 die "skip\n" if !$hotplug_features->{memory};
4238                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4239             } elsif ($opt eq 'cpuunits') {
4240                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4241             } elsif ($opt eq 'cpulimit') {
4242                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4243                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4244             } else {
4245                 die "skip\n";  # skip non-hot-pluggable options
4246             }
4247         };
4248         if (my $err = $@) {
4249             &$add_error($opt, $err) if $err ne "skip\n";
4250         } else {
4251             # save new config if hotplug was successful
4252             $conf->{$opt} = $value;
4253             delete $conf->{pending}->{$opt};
4254             PVE::QemuConfig->write_config($vmid, $conf);
4255             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4256         }
4257     }
4258 }
4259
4260 sub try_deallocate_drive {
4261     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4262
4263     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4264         my $volid = $drive->{file};
4265         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4266             my $sid = PVE::Storage::parse_volume_id($volid);
4267             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4268
4269             # check if the disk is really unused
4270             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4271                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4272             PVE::Storage::vdisk_free($storecfg, $volid);
4273             return 1;
4274         } else {
4275             # If vm is not owner of this disk remove from config
4276             return 1;
4277         }
4278     }
4279
4280     return undef;
4281 }
4282
4283 sub vmconfig_delete_or_detach_drive {
4284     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4285
4286     my $drive = parse_drive($opt, $conf->{$opt});
4287
4288     my $rpcenv = PVE::RPCEnvironment::get();
4289     my $authuser = $rpcenv->get_user();
4290
4291     if ($force) {
4292         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4293         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4294     } else {
4295         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4296     }
4297 }
4298
4299 sub vmconfig_apply_pending {
4300     my ($vmid, $conf, $storecfg) = @_;
4301
4302     # cold plug
4303
4304     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4305     while (my ($opt, $force) = each %$pending_delete_hash) {
4306         die "internal error" if $opt =~ m/^unused/;
4307         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4308         if (!defined($conf->{$opt})) {
4309             vmconfig_undelete_pending_option($conf, $opt);
4310             PVE::QemuConfig->write_config($vmid, $conf);
4311         } elsif (is_valid_drivename($opt)) {
4312             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4313             vmconfig_undelete_pending_option($conf, $opt);
4314             delete $conf->{$opt};
4315             PVE::QemuConfig->write_config($vmid, $conf);
4316         } else {
4317             vmconfig_undelete_pending_option($conf, $opt);
4318             delete $conf->{$opt};
4319             PVE::QemuConfig->write_config($vmid, $conf);
4320         }
4321     }
4322
4323     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4324
4325     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4326         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4327
4328         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4329             # skip if nothing changed
4330         } elsif (is_valid_drivename($opt)) {
4331             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4332                 if defined($conf->{$opt});
4333             $conf->{$opt} = $conf->{pending}->{$opt};
4334         } else {
4335             $conf->{$opt} = $conf->{pending}->{$opt};
4336         }
4337
4338         delete $conf->{pending}->{$opt};
4339         PVE::QemuConfig->write_config($vmid, $conf);
4340     }
4341 }
4342
4343 my $safe_num_ne = sub {
4344     my ($a, $b) = @_;
4345
4346     return 0 if !defined($a) && !defined($b);
4347     return 1 if !defined($a);
4348     return 1 if !defined($b);
4349
4350     return $a != $b;
4351 };
4352
4353 my $safe_string_ne = sub {
4354     my ($a, $b) = @_;
4355
4356     return 0 if !defined($a) && !defined($b);
4357     return 1 if !defined($a);
4358     return 1 if !defined($b);
4359
4360     return $a ne $b;
4361 };
4362
4363 sub vmconfig_update_net {
4364     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4365
4366     my $newnet = parse_net($value);
4367
4368     if ($conf->{$opt}) {
4369         my $oldnet = parse_net($conf->{$opt});
4370
4371         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4372             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4373             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4374             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4375
4376             # for non online change, we try to hot-unplug
4377             die "skip\n" if !$hotplug;
4378             vm_deviceunplug($vmid, $conf, $opt);
4379         } else {
4380
4381             die "internal error" if $opt !~ m/net(\d+)/;
4382             my $iface = "tap${vmid}i$1";
4383
4384             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4385                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4386                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4387                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4388                 PVE::Network::tap_unplug($iface);
4389                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4390             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4391                 # Rate can be applied on its own but any change above needs to
4392                 # include the rate in tap_plug since OVS resets everything.
4393                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4394             }
4395
4396             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4397                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4398             }
4399
4400             return 1;
4401         }
4402     }
4403
4404     if ($hotplug) {
4405         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4406     } else {
4407         die "skip\n";
4408     }
4409 }
4410
4411 sub vmconfig_update_disk {
4412     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4413
4414     # fixme: do we need force?
4415
4416     my $drive = parse_drive($opt, $value);
4417
4418     if ($conf->{$opt}) {
4419
4420         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4421
4422             my $media = $drive->{media} || 'disk';
4423             my $oldmedia = $old_drive->{media} || 'disk';
4424             die "unable to change media type\n" if $media ne $oldmedia;
4425
4426             if (!drive_is_cdrom($old_drive)) {
4427
4428                 if ($drive->{file} ne $old_drive->{file}) {
4429
4430                     die "skip\n" if !$hotplug;
4431
4432                     # unplug and register as unused
4433                     vm_deviceunplug($vmid, $conf, $opt);
4434                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4435
4436                 } else {
4437                     # update existing disk
4438
4439                     # skip non hotpluggable value
4440                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4441                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4442                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4443                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4444                         die "skip\n";
4445                     }
4446
4447                     # apply throttle
4448                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4449                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4450                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4451                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4452                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4453                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4454                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4455                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4456                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4457                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4458                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4459                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4460                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4461                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4462                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4463                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4464                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4465                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4466
4467                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4468                                                    ($drive->{mbps} || 0)*1024*1024,
4469                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4470                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4471                                                    $drive->{iops} || 0,
4472                                                    $drive->{iops_rd} || 0,
4473                                                    $drive->{iops_wr} || 0,
4474                                                    ($drive->{mbps_max} || 0)*1024*1024,
4475                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4476                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4477                                                    $drive->{iops_max} || 0,
4478                                                    $drive->{iops_rd_max} || 0,
4479                                                    $drive->{iops_wr_max} || 0,
4480                                                    $drive->{bps_max_length} || 1,
4481                                                    $drive->{bps_rd_max_length} || 1,
4482                                                    $drive->{bps_wr_max_length} || 1,
4483                                                    $drive->{iops_max_length} || 1,
4484                                                    $drive->{iops_rd_max_length} || 1,
4485                                                    $drive->{iops_wr_max_length} || 1);
4486
4487                     }
4488
4489                     return 1;
4490                 }
4491
4492             } else { # cdrom
4493
4494                 if ($drive->{file} eq 'none') {
4495                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4496                 } else {
4497                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4498                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4499                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4500                 }
4501
4502                 return 1;
4503             }
4504         }
4505     }
4506
4507     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4508     # hotplug new disks
4509     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4510     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4511 }
4512
4513 sub vm_start {
4514     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4515         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4516
4517     PVE::QemuConfig->lock_config($vmid, sub {
4518         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4519
4520         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4521
4522         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4523
4524         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4525
4526         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4527             vmconfig_apply_pending($vmid, $conf, $storecfg);
4528             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4529         }
4530
4531         my $defaults = load_defaults();
4532
4533         # set environment variable useful inside network script
4534         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4535
4536         my $local_volumes = {};
4537
4538         if ($targetstorage) {
4539             foreach_drive($conf, sub {
4540                 my ($ds, $drive) = @_;
4541
4542                 return if drive_is_cdrom($drive);
4543
4544                 my $volid = $drive->{file};
4545
4546                 return if !$volid;
4547
4548                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4549
4550                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4551                 return if $scfg->{shared};
4552                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4553             });
4554
4555             my $format = undef;
4556
4557             foreach my $opt (sort keys %$local_volumes) {
4558
4559                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4560                 my $drive = parse_drive($opt, $conf->{$opt});
4561
4562                 #if remote storage is specified, use default format
4563                 if ($targetstorage && $targetstorage ne "1") {
4564                     $storeid = $targetstorage;
4565                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4566                     $format = $defFormat;
4567                 } else {
4568                     #else we use same format than original
4569                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4570                     $format = qemu_img_format($scfg, $volid);
4571                 }
4572
4573                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4574                 my $newdrive = $drive;
4575                 $newdrive->{format} = $format;
4576                 $newdrive->{file} = $newvolid;
4577                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4578                 $local_volumes->{$opt} = $drivestr;
4579                 #pass drive to conf for command line
4580                 $conf->{$opt} = $drivestr;
4581             }
4582         }
4583
4584         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4585
4586         my $migrate_port = 0;
4587         my $migrate_uri;
4588         if ($statefile) {
4589             if ($statefile eq 'tcp') {
4590                 my $localip = "localhost";
4591                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4592                 my $nodename = PVE::INotify::nodename();
4593
4594                 if (!defined($migration_type)) {
4595                     if (defined($datacenterconf->{migration}->{type})) {
4596                         $migration_type = $datacenterconf->{migration}->{type};
4597                     } else {
4598                         $migration_type = 'secure';
4599                     }
4600                 }
4601
4602                 if ($migration_type eq 'insecure') {
4603                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4604                     if ($migrate_network_addr) {
4605                         $localip = $migrate_network_addr;
4606                     } else {
4607                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4608                     }
4609
4610                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4611                 }
4612
4613                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4614                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4615                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4616                 push @$cmd, '-incoming', $migrate_uri;
4617                 push @$cmd, '-S';
4618
4619             } elsif ($statefile eq 'unix') {
4620                 # should be default for secure migrations as a ssh TCP forward
4621                 # tunnel is not deterministic reliable ready and fails regurarly
4622                 # to set up in time, so use UNIX socket forwards
4623                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4624                 unlink $socket_addr;
4625
4626                 $migrate_uri = "unix:$socket_addr";
4627
4628                 push @$cmd, '-incoming', $migrate_uri;
4629                 push @$cmd, '-S';
4630
4631             } else {
4632                 push @$cmd, '-loadstate', $statefile;
4633             }
4634         } elsif ($paused) {
4635             push @$cmd, '-S';
4636         }
4637
4638         # host pci devices
4639         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4640           my $d = parse_hostpci($conf->{"hostpci$i"});
4641           next if !$d;
4642           my $pcidevices = $d->{pciid};
4643           foreach my $pcidevice (@$pcidevices) {
4644                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4645
4646                 my $info = pci_device_info("0000:$pciid");
4647                 die "IOMMU not present\n" if !check_iommu_support();
4648                 die "no pci device info for device '$pciid'\n" if !$info;
4649                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4650                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4651           }
4652         }
4653
4654         PVE::Storage::activate_volumes($storecfg, $vollist);
4655
4656         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4657             my $cmd = [];
4658             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4659             eval  { run_command($cmd); };
4660         }
4661
4662         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4663                                                   : $defaults->{cpuunits};
4664
4665         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4666
4667         my %properties = (
4668             Slice => 'qemu.slice',
4669             KillMode => 'none',
4670             CPUShares => $cpuunits
4671         );
4672
4673         if (my $cpulimit = $conf->{cpulimit}) {
4674             $properties{CPUQuota} = int($cpulimit * 100);
4675         }
4676         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4677
4678         if ($conf->{hugepages}) {
4679
4680             my $code = sub {
4681                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4682                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4683
4684                 PVE::QemuServer::Memory::hugepages_mount();
4685                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4686
4687                 eval  {
4688                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4689                     run_command($cmd, %run_params);
4690                 };
4691
4692                 if (my $err = $@) {
4693                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4694                     die $err;
4695                 }
4696
4697                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4698             };
4699             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4700
4701         } else {
4702             eval  {
4703                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4704                 run_command($cmd, %run_params);
4705             };
4706         }
4707
4708         if (my $err = $@) {
4709             # deactivate volumes if start fails
4710             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4711             die "start failed: $err";
4712         }
4713
4714         print "migration listens on $migrate_uri\n" if $migrate_uri;
4715
4716         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4717             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4718             warn $@ if $@;
4719         }
4720
4721         #start nbd server for storage migration
4722         if ($targetstorage) {
4723             my $nodename = PVE::INotify::nodename();
4724             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4725             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4726             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4727             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4728
4729             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4730
4731             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4732
4733             foreach my $opt (sort keys %$local_volumes) {
4734                 my $volid = $local_volumes->{$opt};
4735                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4736                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4737                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4738             }
4739         }
4740
4741         if ($migratedfrom) {
4742             eval {
4743                 set_migration_caps($vmid);
4744             };
4745             warn $@ if $@;
4746
4747             if ($spice_port) {
4748                 print "spice listens on port $spice_port\n";
4749                 if ($spice_ticket) {
4750                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4751                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4752                 }
4753             }
4754
4755         } else {
4756             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4757                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4758                     if $conf->{balloon};
4759             }
4760
4761             foreach my $opt (keys %$conf) {
4762                 next if $opt !~  m/^net\d+$/;
4763                 my $nicconf = parse_net($conf->{$opt});
4764                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4765             }
4766         }
4767
4768         vm_mon_cmd_nocheck($vmid, 'qom-set',
4769                     path => "machine/peripheral/balloon0",
4770                     property => "guest-stats-polling-interval",
4771                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4772
4773     });
4774 }
4775
4776 sub vm_mon_cmd {
4777     my ($vmid, $execute, %params) = @_;
4778
4779     my $cmd = { execute => $execute, arguments => \%params };
4780     vm_qmp_command($vmid, $cmd);
4781 }
4782
4783 sub vm_mon_cmd_nocheck {
4784     my ($vmid, $execute, %params) = @_;
4785
4786     my $cmd = { execute => $execute, arguments => \%params };
4787     vm_qmp_command($vmid, $cmd, 1);
4788 }
4789
4790 sub vm_qmp_command {
4791     my ($vmid, $cmd, $nocheck) = @_;
4792
4793     my $res;
4794
4795     my $timeout;
4796     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4797         $timeout = $cmd->{arguments}->{timeout};
4798         delete $cmd->{arguments}->{timeout};
4799     }
4800
4801     eval {
4802         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4803         my $sname = qmp_socket($vmid);
4804         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4805             my $qmpclient = PVE::QMPClient->new();
4806
4807             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4808         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4809             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4810                 if scalar(%{$cmd->{arguments}});
4811             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4812         } else {
4813             die "unable to open monitor socket\n";
4814         }
4815     };
4816     if (my $err = $@) {
4817         syslog("err", "VM $vmid qmp command failed - $err");
4818         die $err;
4819     }
4820
4821     return $res;
4822 }
4823
4824 sub vm_human_monitor_command {
4825     my ($vmid, $cmdline) = @_;
4826
4827     my $res;
4828
4829     my $cmd = {
4830         execute => 'human-monitor-command',
4831         arguments => { 'command-line' => $cmdline},
4832     };
4833
4834     return vm_qmp_command($vmid, $cmd);
4835 }
4836
4837 sub vm_commandline {
4838     my ($storecfg, $vmid) = @_;
4839
4840     my $conf = PVE::QemuConfig->load_config($vmid);
4841
4842     my $defaults = load_defaults();
4843
4844     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4845
4846     return PVE::Tools::cmd2string($cmd);
4847 }
4848
4849 sub vm_reset {
4850     my ($vmid, $skiplock) = @_;
4851
4852     PVE::QemuConfig->lock_config($vmid, sub {
4853
4854         my $conf = PVE::QemuConfig->load_config($vmid);
4855
4856         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4857
4858         vm_mon_cmd($vmid, "system_reset");
4859     });
4860 }
4861
4862 sub get_vm_volumes {
4863     my ($conf) = @_;
4864
4865     my $vollist = [];
4866     foreach_volid($conf, sub {
4867         my ($volid, $attr) = @_;
4868
4869         return if $volid =~ m|^/|;
4870
4871         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4872         return if !$sid;
4873
4874         push @$vollist, $volid;
4875     });
4876
4877     return $vollist;
4878 }
4879
4880 sub vm_stop_cleanup {
4881     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4882
4883     eval {
4884
4885         if (!$keepActive) {
4886             my $vollist = get_vm_volumes($conf);
4887             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4888         }
4889
4890         foreach my $ext (qw(mon qmp pid vnc qga)) {
4891             unlink "/var/run/qemu-server/${vmid}.$ext";
4892         }
4893
4894         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4895     };
4896     warn $@ if $@; # avoid errors - just warn
4897 }
4898
4899 # Note: use $nockeck to skip tests if VM configuration file exists.
4900 # We need that when migration VMs to other nodes (files already moved)
4901 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4902 sub vm_stop {
4903     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4904
4905     $force = 1 if !defined($force) && !$shutdown;
4906
4907     if ($migratedfrom){
4908         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4909         kill 15, $pid if $pid;
4910         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4911         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4912         return;
4913     }
4914
4915     PVE::QemuConfig->lock_config($vmid, sub {
4916
4917         my $pid = check_running($vmid, $nocheck);
4918         return if !$pid;
4919
4920         my $conf;
4921         if (!$nocheck) {
4922             $conf = PVE::QemuConfig->load_config($vmid);
4923             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4924             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4925                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4926                 $timeout = $opts->{down} if $opts->{down};
4927             }
4928         }
4929
4930         $timeout = 60 if !defined($timeout);
4931
4932         eval {
4933             if ($shutdown) {
4934                 if (defined($conf) && $conf->{agent}) {
4935                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4936                 } else {
4937                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4938                 }
4939             } else {
4940                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4941             }
4942         };
4943         my $err = $@;
4944
4945         if (!$err) {
4946             my $count = 0;
4947             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4948                 $count++;
4949                 sleep 1;
4950             }
4951
4952             if ($count >= $timeout) {
4953                 if ($force) {
4954                     warn "VM still running - terminating now with SIGTERM\n";
4955                     kill 15, $pid;
4956                 } else {
4957                     die "VM quit/powerdown failed - got timeout\n";
4958                 }
4959             } else {
4960                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4961                 return;
4962             }
4963         } else {
4964             if ($force) {
4965                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4966                 kill 15, $pid;
4967             } else {
4968                 die "VM quit/powerdown failed\n";
4969             }
4970         }
4971
4972         # wait again
4973         $timeout = 10;
4974
4975         my $count = 0;
4976         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4977             $count++;
4978             sleep 1;
4979         }
4980
4981         if ($count >= $timeout) {
4982             warn "VM still running - terminating now with SIGKILL\n";
4983             kill 9, $pid;
4984             sleep 1;
4985         }
4986
4987         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4988    });
4989 }
4990
4991 sub vm_suspend {
4992     my ($vmid, $skiplock) = @_;
4993
4994     PVE::QemuConfig->lock_config($vmid, sub {
4995
4996         my $conf = PVE::QemuConfig->load_config($vmid);
4997
4998         PVE::QemuConfig->check_lock($conf)
4999             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5000
5001         vm_mon_cmd($vmid, "stop");
5002     });
5003 }
5004
5005 sub vm_resume {
5006     my ($vmid, $skiplock, $nocheck) = @_;
5007
5008     PVE::QemuConfig->lock_config($vmid, sub {
5009
5010         if (!$nocheck) {
5011
5012             my $conf = PVE::QemuConfig->load_config($vmid);
5013
5014             PVE::QemuConfig->check_lock($conf)
5015                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5016
5017             vm_mon_cmd($vmid, "cont");
5018
5019         } else {
5020             vm_mon_cmd_nocheck($vmid, "cont");
5021         }
5022     });
5023 }
5024
5025 sub vm_sendkey {
5026     my ($vmid, $skiplock, $key) = @_;
5027
5028     PVE::QemuConfig->lock_config($vmid, sub {
5029
5030         my $conf = PVE::QemuConfig->load_config($vmid);
5031
5032         # there is no qmp command, so we use the human monitor command
5033         vm_human_monitor_command($vmid, "sendkey $key");
5034     });
5035 }
5036
5037 sub vm_destroy {
5038     my ($storecfg, $vmid, $skiplock) = @_;
5039
5040     PVE::QemuConfig->lock_config($vmid, sub {
5041
5042         my $conf = PVE::QemuConfig->load_config($vmid);
5043
5044         if (!check_running($vmid)) {
5045             destroy_vm($storecfg, $vmid, undef, $skiplock);
5046         } else {
5047             die "VM $vmid is running - destroy failed\n";
5048         }
5049     });
5050 }
5051
5052 # pci helpers
5053
5054 sub file_write {
5055     my ($filename, $buf) = @_;
5056
5057     my $fh = IO::File->new($filename, "w");
5058     return undef if !$fh;
5059
5060     my $res = print $fh $buf;
5061
5062     $fh->close();
5063
5064     return $res;
5065 }
5066
5067 sub pci_device_info {
5068     my ($name) = @_;
5069
5070     my $res;
5071
5072     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5073     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5074
5075     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5076     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5077
5078     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5079     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5080
5081     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5082     return undef if !defined($product) || $product !~ s/^0x//;
5083
5084     $res = {
5085         name => $name,
5086         vendor => $vendor,
5087         product => $product,
5088         domain => $domain,
5089         bus => $bus,
5090         slot => $slot,
5091         func => $func,
5092         irq => $irq,
5093         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5094     };
5095
5096     return $res;
5097 }
5098
5099 sub pci_dev_reset {
5100     my ($dev) = @_;
5101
5102     my $name = $dev->{name};
5103
5104     my $fn = "$pcisysfs/devices/$name/reset";
5105
5106     return file_write($fn, "1");
5107 }
5108
5109 sub pci_dev_bind_to_vfio {
5110     my ($dev) = @_;
5111
5112     my $name = $dev->{name};
5113
5114     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5115
5116     if (!-d $vfio_basedir) {
5117         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5118     }
5119     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5120
5121     my $testdir = "$vfio_basedir/$name";
5122     return 1 if -d $testdir;
5123
5124     my $data = "$dev->{vendor} $dev->{product}";
5125     return undef if !file_write("$vfio_basedir/new_id", $data);
5126
5127     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5128     if (!file_write($fn, $name)) {
5129         return undef if -f $fn;
5130     }
5131
5132     $fn = "$vfio_basedir/bind";
5133     if (! -d $testdir) {
5134         return undef if !file_write($fn, $name);
5135     }
5136
5137     return -d $testdir;
5138 }
5139
5140 sub pci_dev_group_bind_to_vfio {
5141     my ($pciid) = @_;
5142
5143     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5144
5145     if (!-d $vfio_basedir) {
5146         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5147     }
5148     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5149
5150     # get IOMMU group devices
5151     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5152       my @devs = grep /^0000:/, readdir($D);
5153     closedir($D);
5154
5155     foreach my $pciid (@devs) {
5156         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5157
5158         # pci bridges, switches or root ports are not supported
5159         # they have a pci_bus subdirectory so skip them
5160         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5161
5162         my $info = pci_device_info($1);
5163         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5164     }
5165
5166     return 1;
5167 }
5168
5169 # vzdump restore implementaion
5170
5171 sub tar_archive_read_firstfile {
5172     my $archive = shift;
5173
5174     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5175
5176     # try to detect archive type first
5177     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5178         die "unable to open file '$archive'\n";
5179     my $firstfile = <$fh>;
5180     kill 15, $pid;
5181     close $fh;
5182
5183     die "ERROR: archive contaions no data\n" if !$firstfile;
5184     chomp $firstfile;
5185
5186     return $firstfile;
5187 }
5188
5189 sub tar_restore_cleanup {
5190     my ($storecfg, $statfile) = @_;
5191
5192     print STDERR "starting cleanup\n";
5193
5194     if (my $fd = IO::File->new($statfile, "r")) {
5195         while (defined(my $line = <$fd>)) {
5196             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5197                 my $volid = $2;
5198                 eval {
5199                     if ($volid =~ m|^/|) {
5200                         unlink $volid || die 'unlink failed\n';
5201                     } else {
5202                         PVE::Storage::vdisk_free($storecfg, $volid);
5203                     }
5204                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5205                 };
5206                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5207             } else {
5208                 print STDERR "unable to parse line in statfile - $line";
5209             }
5210         }
5211         $fd->close();
5212     }
5213 }
5214
5215 sub restore_archive {
5216     my ($archive, $vmid, $user, $opts) = @_;
5217
5218     my $format = $opts->{format};
5219     my $comp;
5220
5221     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5222         $format = 'tar' if !$format;
5223         $comp = 'gzip';
5224     } elsif ($archive =~ m/\.tar$/) {
5225         $format = 'tar' if !$format;
5226     } elsif ($archive =~ m/.tar.lzo$/) {
5227         $format = 'tar' if !$format;
5228         $comp = 'lzop';
5229     } elsif ($archive =~ m/\.vma$/) {
5230         $format = 'vma' if !$format;
5231     } elsif ($archive =~ m/\.vma\.gz$/) {
5232         $format = 'vma' if !$format;
5233         $comp = 'gzip';
5234     } elsif ($archive =~ m/\.vma\.lzo$/) {
5235         $format = 'vma' if !$format;
5236         $comp = 'lzop';
5237     } else {
5238         $format = 'vma' if !$format; # default
5239     }
5240
5241     # try to detect archive format
5242     if ($format eq 'tar') {
5243         return restore_tar_archive($archive, $vmid, $user, $opts);
5244     } else {
5245         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5246     }
5247 }
5248
5249 sub restore_update_config_line {
5250     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5251
5252     return if $line =~ m/^\#qmdump\#/;
5253     return if $line =~ m/^\#vzdump\#/;
5254     return if $line =~ m/^lock:/;
5255     return if $line =~ m/^unused\d+:/;
5256     return if $line =~ m/^parent:/;
5257     return if $line =~ m/^template:/; # restored VM is never a template
5258
5259     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5260     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5261         # try to convert old 1.X settings
5262         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5263         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5264             my ($model, $macaddr) = split(/\=/, $devconfig);
5265             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5266             my $net = {
5267                 model => $model,
5268                 bridge => "vmbr$ind",
5269                 macaddr => $macaddr,
5270             };
5271             my $netstr = print_net($net);
5272
5273             print $outfd "net$cookie->{netcount}: $netstr\n";
5274             $cookie->{netcount}++;
5275         }
5276     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5277         my ($id, $netstr) = ($1, $2);
5278         my $net = parse_net($netstr);
5279         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5280         $netstr = print_net($net);
5281         print $outfd "$id: $netstr\n";
5282     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5283         my $virtdev = $1;
5284         my $value = $3;
5285         my $di = parse_drive($virtdev, $value);
5286         if (defined($di->{backup}) && !$di->{backup}) {
5287             print $outfd "#$line";
5288         } elsif ($map->{$virtdev}) {
5289             delete $di->{format}; # format can change on restore
5290             $di->{file} = $map->{$virtdev};
5291             $value = print_drive($vmid, $di);
5292             print $outfd "$virtdev: $value\n";
5293         } else {
5294             print $outfd $line;
5295         }
5296     } else {
5297         print $outfd $line;
5298     }
5299 }
5300
5301 sub scan_volids {
5302     my ($cfg, $vmid) = @_;
5303
5304     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5305
5306     my $volid_hash = {};
5307     foreach my $storeid (keys %$info) {
5308         foreach my $item (@{$info->{$storeid}}) {
5309             next if !($item->{volid} && $item->{size});
5310             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5311             $volid_hash->{$item->{volid}} = $item;
5312         }
5313     }
5314
5315     return $volid_hash;
5316 }
5317
5318 sub is_volume_in_use {
5319     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5320
5321     my $path = PVE::Storage::path($storecfg, $volid);
5322
5323     my $scan_config = sub {
5324         my ($cref, $snapname) = @_;
5325
5326         foreach my $key (keys %$cref) {
5327             my $value = $cref->{$key};
5328             if (is_valid_drivename($key)) {
5329                 next if $skip_drive && $key eq $skip_drive;
5330                 my $drive = parse_drive($key, $value);
5331                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5332                 return 1 if $volid eq $drive->{file};
5333                 if ($drive->{file} =~ m!^/!) {
5334                     return 1 if $drive->{file} eq $path;
5335                 } else {
5336                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5337                     next if !$storeid;
5338                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5339                     next if !$scfg;
5340                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5341                 }
5342             }
5343         }
5344
5345         return 0;
5346     };
5347
5348     return 1 if &$scan_config($conf);
5349
5350     undef $skip_drive;
5351
5352     foreach my $snapname (keys %{$conf->{snapshots}}) {
5353         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5354     }
5355
5356     return 0;
5357 }
5358
5359 sub update_disksize {
5360     my ($vmid, $conf, $volid_hash) = @_;
5361
5362     my $changes;
5363
5364     # used and unused disks
5365     my $referenced = {};
5366
5367     # Note: it is allowed to define multiple storages with same path (alias), so
5368     # we need to check both 'volid' and real 'path' (two different volid can point
5369     # to the same path).
5370
5371     my $referencedpath = {};
5372
5373     # update size info
5374     foreach my $opt (keys %$conf) {
5375         if (is_valid_drivename($opt)) {
5376             my $drive = parse_drive($opt, $conf->{$opt});
5377             my $volid = $drive->{file};
5378             next if !$volid;
5379
5380             $referenced->{$volid} = 1;
5381             if ($volid_hash->{$volid} &&
5382                 (my $path = $volid_hash->{$volid}->{path})) {
5383                 $referencedpath->{$path} = 1;
5384             }
5385
5386             next if drive_is_cdrom($drive);
5387             next if !$volid_hash->{$volid};
5388
5389             $drive->{size} = $volid_hash->{$volid}->{size};
5390             my $new = print_drive($vmid, $drive);
5391             if ($new ne $conf->{$opt}) {
5392                 $changes = 1;
5393                 $conf->{$opt} = $new;
5394             }
5395         }
5396     }
5397
5398     # remove 'unusedX' entry if volume is used
5399     foreach my $opt (keys %$conf) {
5400         next if $opt !~ m/^unused\d+$/;
5401         my $volid = $conf->{$opt};
5402         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5403         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5404             $changes = 1;
5405             delete $conf->{$opt};
5406         }
5407
5408         $referenced->{$volid} = 1;
5409         $referencedpath->{$path} = 1 if $path;
5410     }
5411
5412     foreach my $volid (sort keys %$volid_hash) {
5413         next if $volid =~ m/vm-$vmid-state-/;
5414         next if $referenced->{$volid};
5415         my $path = $volid_hash->{$volid}->{path};
5416         next if !$path; # just to be sure
5417         next if $referencedpath->{$path};
5418         $changes = 1;
5419         PVE::QemuConfig->add_unused_volume($conf, $volid);
5420         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5421     }
5422
5423     return $changes;
5424 }
5425
5426 sub rescan {
5427     my ($vmid, $nolock) = @_;
5428
5429     my $cfg = PVE::Storage::config();
5430
5431     my $volid_hash = scan_volids($cfg, $vmid);
5432
5433     my $updatefn =  sub {
5434         my ($vmid) = @_;
5435
5436         my $conf = PVE::QemuConfig->load_config($vmid);
5437
5438         PVE::QemuConfig->check_lock($conf);
5439
5440         my $vm_volids = {};
5441         foreach my $volid (keys %$volid_hash) {
5442             my $info = $volid_hash->{$volid};
5443             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5444         }
5445
5446         my $changes = update_disksize($vmid, $conf, $vm_volids);
5447
5448         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5449     };
5450
5451     if (defined($vmid)) {
5452         if ($nolock) {
5453             &$updatefn($vmid);
5454         } else {
5455             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5456         }
5457     } else {
5458         my $vmlist = config_list();
5459         foreach my $vmid (keys %$vmlist) {
5460             if ($nolock) {
5461                 &$updatefn($vmid);
5462             } else {
5463                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5464             }
5465         }
5466     }
5467 }
5468
5469 sub restore_vma_archive {
5470     my ($archive, $vmid, $user, $opts, $comp) = @_;
5471
5472     my $input = $archive eq '-' ? "<&STDIN" : undef;
5473     my $readfrom = $archive;
5474
5475     my $uncomp = '';
5476     if ($comp) {
5477         $readfrom = '-';
5478         my $qarchive = PVE::Tools::shellquote($archive);
5479         if ($comp eq 'gzip') {
5480             $uncomp = "zcat $qarchive|";
5481         } elsif ($comp eq 'lzop') {
5482             $uncomp = "lzop -d -c $qarchive|";
5483         } else {
5484             die "unknown compression method '$comp'\n";
5485         }
5486
5487     }
5488
5489     my $tmpdir = "/var/tmp/vzdumptmp$$";
5490     rmtree $tmpdir;
5491
5492     # disable interrupts (always do cleanups)
5493     local $SIG{INT} =
5494         local $SIG{TERM} =
5495         local $SIG{QUIT} =
5496         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5497
5498     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5499     POSIX::mkfifo($mapfifo, 0600);
5500     my $fifofh;
5501
5502     my $openfifo = sub {
5503         open($fifofh, '>', $mapfifo) || die $!;
5504     };
5505
5506     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5507
5508     my $oldtimeout;
5509     my $timeout = 5;
5510
5511     my $devinfo = {};
5512
5513     my $rpcenv = PVE::RPCEnvironment::get();
5514
5515     my $conffile = PVE::QemuConfig->config_file($vmid);
5516     my $tmpfn = "$conffile.$$.tmp";
5517
5518     # Note: $oldconf is undef if VM does not exists
5519     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5520     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5521
5522     my $print_devmap = sub {
5523         my $virtdev_hash = {};
5524
5525         my $cfgfn = "$tmpdir/qemu-server.conf";
5526
5527         # we can read the config - that is already extracted
5528         my $fh = IO::File->new($cfgfn, "r") ||
5529             "unable to read qemu-server.conf - $!\n";
5530
5531         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5532         if (-f $fwcfgfn) {
5533             my $pve_firewall_dir = '/etc/pve/firewall';
5534             mkdir $pve_firewall_dir; # make sure the dir exists
5535             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5536         }
5537
5538         while (defined(my $line = <$fh>)) {
5539             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5540                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5541                 die "archive does not contain data for drive '$virtdev'\n"
5542                     if !$devinfo->{$devname};
5543                 if (defined($opts->{storage})) {
5544                     $storeid = $opts->{storage} || 'local';
5545                 } elsif (!$storeid) {
5546                     $storeid = 'local';
5547                 }
5548                 $format = 'raw' if !$format;
5549                 $devinfo->{$devname}->{devname} = $devname;
5550                 $devinfo->{$devname}->{virtdev} = $virtdev;
5551                 $devinfo->{$devname}->{format} = $format;
5552                 $devinfo->{$devname}->{storeid} = $storeid;
5553
5554                 # check permission on storage
5555                 my $pool = $opts->{pool}; # todo: do we need that?
5556                 if ($user ne 'root@pam') {
5557                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5558                 }
5559
5560                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5561             }
5562         }
5563
5564         foreach my $devname (keys %$devinfo) {
5565             die "found no device mapping information for device '$devname'\n"
5566                 if !$devinfo->{$devname}->{virtdev};
5567         }
5568
5569         my $cfg = PVE::Storage::config();
5570
5571         # create empty/temp config
5572         if ($oldconf) {
5573             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5574             foreach_drive($oldconf, sub {
5575                 my ($ds, $drive) = @_;
5576
5577                 return if drive_is_cdrom($drive);
5578
5579                 my $volid = $drive->{file};
5580
5581                 return if !$volid || $volid =~ m|^/|;
5582
5583                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5584                 return if !$path || !$owner || ($owner != $vmid);
5585
5586                 # Note: only delete disk we want to restore
5587                 # other volumes will become unused
5588                 if ($virtdev_hash->{$ds}) {
5589                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5590                     if (my $err = $@) {
5591                         warn $err;
5592                     }
5593                 }
5594             });
5595
5596             # delete vmstate files
5597             # since after the restore we have no snapshots anymore
5598             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5599                 my $snap = $oldconf->{snapshots}->{$snapname};
5600                 if ($snap->{vmstate}) {
5601                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5602                     if (my $err = $@) {
5603                         warn $err;
5604                     }
5605                 }
5606             }
5607         }
5608
5609         my $map = {};
5610         foreach my $virtdev (sort keys %$virtdev_hash) {
5611             my $d = $virtdev_hash->{$virtdev};
5612             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5613             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5614
5615             # test if requested format is supported
5616             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5617             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5618             $d->{format} = $defFormat if !$supported;
5619
5620             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5621                                                   $d->{format}, undef, $alloc_size);
5622             print STDERR "new volume ID is '$volid'\n";
5623             $d->{volid} = $volid;
5624             my $path = PVE::Storage::path($cfg, $volid);
5625
5626             PVE::Storage::activate_volumes($cfg,[$volid]);
5627
5628             my $write_zeros = 1;
5629             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5630                 $write_zeros = 0;
5631             }
5632
5633             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5634
5635             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5636             $map->{$virtdev} = $volid;
5637         }
5638
5639         $fh->seek(0, 0) || die "seek failed - $!\n";
5640
5641         my $outfd = new IO::File ($tmpfn, "w") ||
5642             die "unable to write config for VM $vmid\n";
5643
5644         my $cookie = { netcount => 0 };
5645         while (defined(my $line = <$fh>)) {
5646             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5647         }
5648
5649         $fh->close();
5650         $outfd->close();
5651     };
5652
5653     eval {
5654         # enable interrupts
5655         local $SIG{INT} =
5656             local $SIG{TERM} =
5657             local $SIG{QUIT} =
5658             local $SIG{HUP} =
5659             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5660         local $SIG{ALRM} = sub { die "got timeout\n"; };
5661
5662         $oldtimeout = alarm($timeout);
5663
5664         my $parser = sub {
5665             my $line = shift;
5666
5667             print "$line\n";
5668
5669             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5670                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5671                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5672             } elsif ($line =~ m/^CTIME: /) {
5673                 # we correctly received the vma config, so we can disable
5674                 # the timeout now for disk allocation (set to 10 minutes, so
5675                 # that we always timeout if something goes wrong)
5676                 alarm(600);
5677                 &$print_devmap();
5678                 print $fifofh "done\n";
5679                 my $tmp = $oldtimeout || 0;
5680                 $oldtimeout = undef;
5681                 alarm($tmp);
5682                 close($fifofh);
5683             }
5684         };
5685
5686         print "restore vma archive: $cmd\n";
5687         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5688     };
5689     my $err = $@;
5690
5691     alarm($oldtimeout) if $oldtimeout;
5692
5693     my $vollist = [];
5694     foreach my $devname (keys %$devinfo) {
5695         my $volid = $devinfo->{$devname}->{volid};
5696         push @$vollist, $volid if $volid;
5697     }
5698
5699     my $cfg = PVE::Storage::config();
5700     PVE::Storage::deactivate_volumes($cfg, $vollist);
5701
5702     unlink $mapfifo;
5703
5704     if ($err) {
5705         rmtree $tmpdir;
5706         unlink $tmpfn;
5707
5708         foreach my $devname (keys %$devinfo) {
5709             my $volid = $devinfo->{$devname}->{volid};
5710             next if !$volid;
5711             eval {
5712                 if ($volid =~ m|^/|) {
5713                     unlink $volid || die 'unlink failed\n';
5714                 } else {
5715                     PVE::Storage::vdisk_free($cfg, $volid);
5716                 }
5717                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5718             };
5719             print STDERR "unable to cleanup '$volid' - $@" if $@;
5720         }
5721         die $err;
5722     }
5723
5724     rmtree $tmpdir;
5725
5726     rename($tmpfn, $conffile) ||
5727         die "unable to commit configuration file '$conffile'\n";
5728
5729     PVE::Cluster::cfs_update(); # make sure we read new file
5730
5731     eval { rescan($vmid, 1); };
5732     warn $@ if $@;
5733 }
5734
5735 sub restore_tar_archive {
5736     my ($archive, $vmid, $user, $opts) = @_;
5737
5738     if ($archive ne '-') {
5739         my $firstfile = tar_archive_read_firstfile($archive);
5740         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5741             if $firstfile ne 'qemu-server.conf';
5742     }
5743
5744     my $storecfg = PVE::Storage::config();
5745
5746     # destroy existing data - keep empty config
5747     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5748     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5749
5750     my $tocmd = "/usr/lib/qemu-server/qmextract";
5751
5752     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5753     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5754     $tocmd .= ' --prealloc' if $opts->{prealloc};
5755     $tocmd .= ' --info' if $opts->{info};
5756
5757     # tar option "xf" does not autodetect compression when read from STDIN,
5758     # so we pipe to zcat
5759     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5760         PVE::Tools::shellquote("--to-command=$tocmd");
5761
5762     my $tmpdir = "/var/tmp/vzdumptmp$$";
5763     mkpath $tmpdir;
5764
5765     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5766     local $ENV{VZDUMP_VMID} = $vmid;
5767     local $ENV{VZDUMP_USER} = $user;
5768
5769     my $conffile = PVE::QemuConfig->config_file($vmid);
5770     my $tmpfn = "$conffile.$$.tmp";
5771
5772     # disable interrupts (always do cleanups)
5773     local $SIG{INT} =
5774         local $SIG{TERM} =
5775         local $SIG{QUIT} =
5776         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5777
5778     eval {
5779         # enable interrupts
5780         local $SIG{INT} =
5781             local $SIG{TERM} =
5782             local $SIG{QUIT} =
5783             local $SIG{HUP} =
5784             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5785
5786         if ($archive eq '-') {
5787             print "extracting archive from STDIN\n";
5788             run_command($cmd, input => "<&STDIN");
5789         } else {
5790             print "extracting archive '$archive'\n";
5791             run_command($cmd);
5792         }
5793
5794         return if $opts->{info};
5795
5796         # read new mapping
5797         my $map = {};
5798         my $statfile = "$tmpdir/qmrestore.stat";
5799         if (my $fd = IO::File->new($statfile, "r")) {
5800             while (defined (my $line = <$fd>)) {
5801                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5802                     $map->{$1} = $2 if $1;
5803                 } else {
5804                     print STDERR "unable to parse line in statfile - $line\n";
5805                 }
5806             }
5807             $fd->close();
5808         }
5809
5810         my $confsrc = "$tmpdir/qemu-server.conf";
5811
5812         my $srcfd = new IO::File($confsrc, "r") ||
5813             die "unable to open file '$confsrc'\n";
5814
5815         my $outfd = new IO::File ($tmpfn, "w") ||
5816             die "unable to write config for VM $vmid\n";
5817
5818         my $cookie = { netcount => 0 };
5819         while (defined (my $line = <$srcfd>)) {
5820             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5821         }
5822
5823         $srcfd->close();
5824         $outfd->close();
5825     };
5826     my $err = $@;
5827
5828     if ($err) {
5829
5830         unlink $tmpfn;
5831
5832         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5833
5834         die $err;
5835     }
5836
5837     rmtree $tmpdir;
5838
5839     rename $tmpfn, $conffile ||
5840         die "unable to commit configuration file '$conffile'\n";
5841
5842     PVE::Cluster::cfs_update(); # make sure we read new file
5843
5844     eval { rescan($vmid, 1); };
5845     warn $@ if $@;
5846 };
5847
5848 sub foreach_storage_used_by_vm {
5849     my ($conf, $func) = @_;
5850
5851     my $sidhash = {};
5852
5853     foreach_drive($conf, sub {
5854         my ($ds, $drive) = @_;
5855         return if drive_is_cdrom($drive);
5856
5857         my $volid = $drive->{file};
5858
5859         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5860         $sidhash->{$sid} = $sid if $sid;
5861     });
5862
5863     foreach my $sid (sort keys %$sidhash) {
5864         &$func($sid);
5865     }
5866 }
5867
5868 sub do_snapshots_with_qemu {
5869     my ($storecfg, $volid) = @_;
5870
5871     my $storage_name = PVE::Storage::parse_volume_id($volid);
5872
5873     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5874         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5875         return 1;
5876     }
5877
5878     if ($volid =~ m/\.(qcow2|qed)$/){
5879         return 1;
5880     }
5881
5882     return undef;
5883 }
5884
5885 sub qga_check_running {
5886     my ($vmid) = @_;
5887
5888     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5889     if ($@) {
5890         warn "Qemu Guest Agent is not running - $@";
5891         return 0;
5892     }
5893     return 1;
5894 }
5895
5896 sub template_create {
5897     my ($vmid, $conf, $disk) = @_;
5898
5899     my $storecfg = PVE::Storage::config();
5900
5901     foreach_drive($conf, sub {
5902         my ($ds, $drive) = @_;
5903
5904         return if drive_is_cdrom($drive);
5905         return if $disk && $ds ne $disk;
5906
5907         my $volid = $drive->{file};
5908         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5909
5910         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5911         $drive->{file} = $voliddst;
5912         $conf->{$ds} = print_drive($vmid, $drive);
5913         PVE::QemuConfig->write_config($vmid, $conf);
5914     });
5915 }
5916
5917 sub qemu_img_convert {
5918     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5919
5920     my $storecfg = PVE::Storage::config();
5921     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5922     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5923
5924     if ($src_storeid && $dst_storeid) {
5925
5926         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5927
5928         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5929         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5930
5931         my $src_format = qemu_img_format($src_scfg, $src_volname);
5932         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5933
5934         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5935         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5936
5937         my $cmd = [];
5938         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5939         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5940         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5941         if ($is_zero_initialized) {
5942             push @$cmd, "zeroinit:$dst_path";
5943         } else {
5944             push @$cmd, $dst_path;
5945         }
5946
5947         my $parser = sub {
5948             my $line = shift;
5949             if($line =~ m/\((\S+)\/100\%\)/){
5950                 my $percent = $1;
5951                 my $transferred = int($size * $percent / 100);
5952                 my $remaining = $size - $transferred;
5953
5954                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5955             }
5956
5957         };
5958
5959         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5960         my $err = $@;
5961         die "copy failed: $err" if $err;
5962     }
5963 }
5964
5965 sub qemu_img_format {
5966     my ($scfg, $volname) = @_;
5967
5968     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5969         return $1;
5970     } else {
5971         return "raw";
5972     }
5973 }
5974
5975 sub qemu_drive_mirror {
5976     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
5977
5978     $jobs = {} if !$jobs;
5979
5980     my $qemu_target;
5981     my $format;
5982     $jobs->{"drive-$drive"} = {};
5983
5984     if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
5985         my $server = $1;
5986         my $port = $2;
5987         my $exportname = $3;
5988
5989         $format = "nbd";
5990         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
5991         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
5992         my $cmd = ['socat', '-T30', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=5"];
5993
5994         my $pid = fork();
5995         if (!defined($pid)) {
5996             die "forking socat tunnel failed\n";
5997         } elsif ($pid == 0) {
5998             exec(@$cmd);
5999             warn "exec failed: $!\n";
6000             POSIX::_exit(-1);
6001         }
6002         $jobs->{"drive-$drive"}->{pid} = $pid;
6003
6004         my $timeout = 0;
6005         while (!-S $unixsocket) {
6006             die "nbd connection helper timed out\n"
6007                 if $timeout++ > 5;
6008             sleep 1;
6009         }
6010     } else {
6011         my $storecfg = PVE::Storage::config();
6012         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6013
6014         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6015
6016         $format = qemu_img_format($dst_scfg, $dst_volname);
6017
6018         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6019
6020         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6021     }
6022
6023     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6024     $opts->{format} = $format if $format;
6025
6026     print "drive mirror is starting for drive-$drive\n";
6027
6028     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6029
6030     if (my $err = $@) {
6031         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6032         die "mirroring error: $err";
6033     }
6034
6035     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6036 }
6037
6038 sub qemu_drive_mirror_monitor {
6039     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6040
6041     eval {
6042         my $err_complete = 0;
6043
6044         while (1) {
6045             die "storage migration timed out\n" if $err_complete > 300;
6046
6047             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6048
6049             my $running_mirror_jobs = {};
6050             foreach my $stat (@$stats) {
6051                 next if $stat->{type} ne 'mirror';
6052                 $running_mirror_jobs->{$stat->{device}} = $stat;
6053             }
6054
6055             my $readycounter = 0;
6056
6057             foreach my $job (keys %$jobs) {
6058
6059                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6060                     print "$job : finished\n";
6061                     delete $jobs->{$job};
6062                     next;
6063                 }
6064
6065                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6066
6067                 my $busy = $running_mirror_jobs->{$job}->{busy};
6068                 my $ready = $running_mirror_jobs->{$job}->{ready};
6069                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6070                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6071                     my $remaining = $total - $transferred;
6072                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6073
6074                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6075                 }
6076
6077                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6078             }
6079
6080             last if scalar(keys %$jobs) == 0;
6081
6082             if ($readycounter == scalar(keys %$jobs)) {
6083                 print "all mirroring jobs are ready \n";
6084                 last if $skipcomplete; #do the complete later
6085
6086                 if ($vmiddst && $vmiddst != $vmid) {
6087                     my $agent_running = $qga && qga_check_running($vmid);
6088                     if ($agent_running) {
6089                         print "freeze filesystem\n";
6090                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6091                     } else {
6092                         print "suspend vm\n";
6093                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6094                     }
6095
6096                     # if we clone a disk for a new target vm, we don't switch the disk
6097                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6098
6099                     if ($agent_running) {
6100                         print "unfreeze filesystem\n";
6101                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6102                     } else {
6103                         print "resume vm\n";
6104                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6105                     }
6106
6107                     last;
6108                 } else {
6109
6110                     foreach my $job (keys %$jobs) {
6111                         # try to switch the disk if source and destination are on the same guest
6112                         print "$job: Completing block job...\n";
6113
6114                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6115                         if ($@ =~ m/cannot be completed/) {
6116                             print "$job: Block job cannot be completed, try again.\n";
6117                             $err_complete++;
6118                         }else {
6119                             print "$job: Completed successfully.\n";
6120                             $jobs->{$job}->{complete} = 1;
6121                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6122                         }
6123                     }
6124                 }
6125             }
6126             sleep 1;
6127         }
6128     };
6129     my $err = $@;
6130
6131     if ($err) {
6132         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6133         die "mirroring error: $err";
6134     }
6135
6136 }
6137
6138 sub qemu_blockjobs_cancel {
6139     my ($vmid, $jobs) = @_;
6140
6141     foreach my $job (keys %$jobs) {
6142         print "$job: Cancelling block job\n";
6143         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6144         $jobs->{$job}->{cancel} = 1;
6145     }
6146
6147     while (1) {
6148         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6149
6150         my $running_jobs = {};
6151         foreach my $stat (@$stats) {
6152             $running_jobs->{$stat->{device}} = $stat;
6153         }
6154
6155         foreach my $job (keys %$jobs) {
6156
6157             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6158                 print "$job: Done.\n";
6159                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6160                 delete $jobs->{$job};
6161             }
6162         }
6163
6164         last if scalar(keys %$jobs) == 0;
6165
6166         sleep 1;
6167     }
6168 }
6169
6170 sub qemu_blockjobs_finish_tunnel {
6171    my ($vmid, $job, $cpid) = @_;
6172
6173    return if !$cpid;
6174
6175    for (my $i = 1; $i < 20; $i++) {
6176         my $waitpid = waitpid($cpid, WNOHANG);
6177         last if (defined($waitpid) && ($waitpid == $cpid));
6178
6179         if ($i == 10) {
6180             kill(15, $cpid);
6181         } elsif ($i >= 15) {
6182             kill(9, $cpid);
6183         }
6184         sleep (1);
6185     }
6186     unlink "/run/qemu-server/$vmid.mirror-$job";
6187 }
6188
6189 sub clone_disk {
6190     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6191         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6192
6193     my $newvolid;
6194
6195     if (!$full) {
6196         print "create linked clone of drive $drivename ($drive->{file})\n";
6197         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6198         push @$newvollist, $newvolid;
6199     } else {
6200
6201         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6202         $storeid = $storage if $storage;
6203
6204         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6205         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6206
6207         print "create full clone of drive $drivename ($drive->{file})\n";
6208         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, undef, ($size/1024));
6209         push @$newvollist, $newvolid;
6210
6211         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6212
6213         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6214         if (!$running || $snapname) {
6215             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6216         } else {
6217
6218             my $kvmver = get_running_qemu_version ($vmid);
6219             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6220                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6221                     if $drive->{iothread};
6222             }
6223
6224             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6225         }
6226     }
6227
6228     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6229
6230     my $disk = $drive;
6231     $disk->{format} = undef;
6232     $disk->{file} = $newvolid;
6233     $disk->{size} = $size;
6234
6235     return $disk;
6236 }
6237
6238 # this only works if VM is running
6239 sub get_current_qemu_machine {
6240     my ($vmid) = @_;
6241
6242     my $cmd = { execute => 'query-machines', arguments => {} };
6243     my $res = vm_qmp_command($vmid, $cmd);
6244
6245     my ($current, $default);
6246     foreach my $e (@$res) {
6247         $default = $e->{name} if $e->{'is-default'};
6248         $current = $e->{name} if $e->{'is-current'};
6249     }
6250
6251     # fallback to the default machine if current is not supported by qemu
6252     return $current || $default || 'pc';
6253 }
6254
6255 sub get_running_qemu_version {
6256     my ($vmid) = @_;
6257     my $cmd = { execute => 'query-version', arguments => {} };
6258     my $res = vm_qmp_command($vmid, $cmd);
6259     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6260 }
6261
6262 sub qemu_machine_feature_enabled {
6263     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6264
6265     my $current_major;
6266     my $current_minor;
6267
6268     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6269
6270         $current_major = $3;
6271         $current_minor = $4;
6272
6273     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6274
6275         $current_major = $1;
6276         $current_minor = $2;
6277     }
6278
6279     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6280
6281
6282 }
6283
6284 sub qemu_machine_pxe {
6285     my ($vmid, $conf, $machine) = @_;
6286
6287     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6288
6289     foreach my $opt (keys %$conf) {
6290         next if $opt !~ m/^net(\d+)$/;
6291         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6292         next if !$net;
6293         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6294         return $machine.".pxe" if $romfile =~ m/pxe/;
6295         last;
6296     }
6297
6298     return $machine;
6299 }
6300
6301 sub qemu_use_old_bios_files {
6302     my ($machine_type) = @_;
6303
6304     return if !$machine_type;
6305
6306     my $use_old_bios_files = undef;
6307
6308     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6309         $machine_type = $1;
6310         $use_old_bios_files = 1;
6311     } else {
6312         my $kvmver = kvm_user_version();
6313         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6314         # load new efi bios files on migration. So this hack is required to allow
6315         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6316         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6317         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6318     }
6319
6320     return ($use_old_bios_files, $machine_type);
6321 }
6322
6323 sub create_efidisk {
6324     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6325
6326     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6327
6328     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6329     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6330     PVE::Storage::activate_volumes($storecfg, [$volid]);
6331
6332     my $path = PVE::Storage::path($storecfg, $volid);
6333     eval {
6334         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6335     };
6336     die "Copying EFI vars image failed: $@" if $@;
6337
6338     return ($volid, $vars_size);
6339 }
6340
6341 sub lspci {
6342
6343     my $devices = {};
6344
6345     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6346             my (undef, $id, $function) = @_;
6347             my $res = { id => $id, function => $function};
6348             push @{$devices->{$id}}, $res;
6349     });
6350
6351     # Entries should be sorted by functions.
6352     foreach my $id (keys %$devices) {
6353         my $dev = $devices->{$id};
6354         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6355     }
6356
6357     return $devices;
6358 }
6359
6360 sub vm_iothreads_list {
6361     my ($vmid) = @_;
6362
6363     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6364
6365     my $iothreads = {};
6366     foreach my $iothread (@$res) {
6367         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6368     }
6369
6370     return $iothreads;
6371 }
6372
6373 sub scsihw_infos {
6374     my ($conf, $drive) = @_;
6375
6376     my $maxdev = 0;
6377
6378     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6379         $maxdev = 7;
6380     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6381         $maxdev = 1;
6382     } else {
6383         $maxdev = 256;
6384     }
6385
6386     my $controller = int($drive->{index} / $maxdev);
6387     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6388
6389     return ($maxdev, $controller, $controller_prefix);
6390 }
6391
6392 sub add_hyperv_enlightenments {
6393     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6394
6395     return if $winversion < 6;
6396     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6397
6398     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6399
6400     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6401         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6402         push @$cpuFlags , 'hv_vapic';
6403         push @$cpuFlags , 'hv_time';
6404     } else {
6405         push @$cpuFlags , 'hv_spinlocks=0xffff';
6406     }
6407
6408     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6409         push @$cpuFlags , 'hv_reset';
6410         push @$cpuFlags , 'hv_vpindex';
6411         push @$cpuFlags , 'hv_runtime';
6412     }
6413
6414     if ($winversion >= 7) {
6415         push @$cpuFlags , 'hv_relaxed';
6416     }
6417 }
6418
6419 sub windows_version {
6420     my ($ostype) = @_;
6421
6422     return 0 if !$ostype;
6423
6424     my $winversion = 0;
6425
6426     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6427         $winversion = 5;
6428     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6429         $winversion = 6;
6430     } elsif ($ostype =~ m/^win(\d+)$/) {
6431         $winversion = $1;
6432     }
6433
6434     return $winversion;
6435 }
6436
6437 sub resolve_dst_disk_format {
6438         my ($storecfg, $storeid, $src_volname, $format) = @_;
6439         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6440
6441         if (!$format) {
6442             # if no target format is specified, use the source disk format as hint
6443             if ($src_volname) {
6444                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6445                 $format = qemu_img_format($scfg, $src_volname);
6446             } else {
6447                 return $defFormat;
6448             }
6449         }
6450
6451         # test if requested format is supported - else use default
6452         my $supported = grep { $_ eq $format } @$validFormats;
6453         $format = $defFormat if !$supported;
6454         return $format;
6455 }
6456
6457 sub resolve_first_disk {
6458     my $conf = shift;
6459     my @disks = PVE::QemuServer::valid_drive_names();
6460     my $firstdisk;
6461     foreach my $ds (reverse @disks) {
6462         next if !$conf->{$ds};
6463         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6464         next if PVE::QemuServer::drive_is_cdrom($disk);
6465         $firstdisk = $ds;
6466     }
6467     return $firstdisk;
6468 }
6469
6470 sub generate_smbios1_uuid {
6471     my ($uuid, $uuid_str);
6472     UUID::generate($uuid);
6473     UUID::unparse($uuid, $uuid_str);
6474     return "uuid=$uuid_str";
6475 }
6476
6477 # bash completion helper
6478
6479 sub complete_backup_archives {
6480     my ($cmdname, $pname, $cvalue) = @_;
6481
6482     my $cfg = PVE::Storage::config();
6483
6484     my $storeid;
6485
6486     if ($cvalue =~ m/^([^:]+):/) {
6487         $storeid = $1;
6488     }
6489
6490     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6491
6492     my $res = [];
6493     foreach my $id (keys %$data) {
6494         foreach my $item (@{$data->{$id}}) {
6495             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6496             push @$res, $item->{volid} if defined($item->{volid});
6497         }
6498     }
6499
6500     return $res;
6501 }
6502
6503 my $complete_vmid_full = sub {
6504     my ($running) = @_;
6505
6506     my $idlist = vmstatus();
6507
6508     my $res = [];
6509
6510     foreach my $id (keys %$idlist) {
6511         my $d = $idlist->{$id};
6512         if (defined($running)) {
6513             next if $d->{template};
6514             next if $running && $d->{status} ne 'running';
6515             next if !$running && $d->{status} eq 'running';
6516         }
6517         push @$res, $id;
6518
6519     }
6520     return $res;
6521 };
6522
6523 sub complete_vmid {
6524     return &$complete_vmid_full();
6525 }
6526
6527 sub complete_vmid_stopped {
6528     return &$complete_vmid_full(0);
6529 }
6530
6531 sub complete_vmid_running {
6532     return &$complete_vmid_full(1);
6533 }
6534
6535 sub complete_storage {
6536
6537     my $cfg = PVE::Storage::config();
6538     my $ids = $cfg->{ids};
6539
6540     my $res = [];
6541     foreach my $sid (keys %$ids) {
6542         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6543         next if !$ids->{$sid}->{content}->{images};
6544         push @$res, $sid;
6545     }
6546
6547     return $res;
6548 }
6549
6550 sub nbd_stop {
6551     my ($vmid) = @_;
6552
6553     vm_mon_cmd($vmid, 'nbd-server-stop');
6554 }
6555
6556 1;