PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42
  43 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  44
  45 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  46
  47 # Note about locking: we use flock on the config file protect
  48 # against concurent actions.
  49 # Aditionaly, we have a 'lock' setting in the config file. This
  50 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  51 # allowed when such lock is set. But you can ignore this kind of
  52 # lock with the --skiplock flag.
  53
  54 cfs_register_file('/qemu-server/',
  55                   \&parse_vm_config,
  56                   \&write_vm_config);
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  72     type => 'string',
  73     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  74     description => "The drive's backing file's data format.",
  75     optional => 1,
  76 });
  77
  78 #no warnings 'redefine';
  79
  80 sub cgroups_write {
  81    my ($controller, $vmid, $option, $value) = @_;
  82
  83    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  84    PVE::ProcFSTools::write_proc_entry($path, $value);
  85
  86 }
  87
  88 my $nodename = PVE::INotify::nodename();
  89
  90 mkdir "/etc/pve/nodes/$nodename";
  91 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  92 mkdir $confdir;
  93
  94 my $var_run_tmpdir = "/var/run/qemu-server";
  95 mkdir $var_run_tmpdir;
  96
  97 my $lock_dir = "/var/lock/qemu-server";
  98 mkdir $lock_dir;
  99
 100 my $pcisysfs = "/sys/bus/pci";
 101
 102 my $cpu_vendor_list = {
 103     # Intel CPUs
 104     486 => 'GenuineIntel',
 105     pentium => 'GenuineIntel',
 106     pentium2  => 'GenuineIntel',
 107     pentium3  => 'GenuineIntel',
 108     coreduo => 'GenuineIntel',
 109     core2duo => 'GenuineIntel',
 110     Conroe  => 'GenuineIntel',
 111     Penryn  => 'GenuineIntel',
 112     Nehalem  => 'GenuineIntel',
 113     Westmere => 'GenuineIntel',
 114     SandyBridge => 'GenuineIntel',
 115     IvyBridge => 'GenuineIntel',
 116     Haswell => 'GenuineIntel',
 117     'Haswell-noTSX' => 'GenuineIntel',
 118     Broadwell => 'GenuineIntel',
 119     'Broadwell-noTSX' => 'GenuineIntel',
 120     'Skylake-Client' => 'GenuineIntel',
 121
 122     # AMD CPUs
 123     athlon => 'AuthenticAMD',
 124     phenom  => 'AuthenticAMD',
 125     Opteron_G1  => 'AuthenticAMD',
 126     Opteron_G2  => 'AuthenticAMD',
 127     Opteron_G3  => 'AuthenticAMD',
 128     Opteron_G4  => 'AuthenticAMD',
 129     Opteron_G5  => 'AuthenticAMD',
 130
 131     # generic types, use vendor from host node
 132     host => 'default',
 133     kvm32 => 'default',
 134     kvm64 => 'default',
 135     qemu32 => 'default',
 136     qemu64 => 'default',
 137 };
 138
 139 my $cpu_fmt = {
 140     cputype => {
 141         description => "Emulated CPU type.",
 142         type => 'string',
 143         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 144         default => 'kvm64',
 145         default_key => 1,
 146     },
 147     hidden => {
 148         description => "Do not identify as a KVM virtual machine.",
 149         type => 'boolean',
 150         optional => 1,
 151         default => 0
 152     },
 153 };
 154
 155 my $watchdog_fmt = {
 156     model => {
 157         default_key => 1,
 158         type => 'string',
 159         enum => [qw(i6300esb ib700)],
 160         description => "Watchdog type to emulate.",
 161         default => 'i6300esb',
 162         optional => 1,
 163     },
 164     action => {
 165         type => 'string',
 166         enum => [qw(reset shutdown poweroff pause debug none)],
 167         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 168         optional => 1,
 169     },
 170 };
 171 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 172
 173 my $confdesc = {
 174     onboot => {
 175         optional => 1,
 176         type => 'boolean',
 177         description => "Specifies whether a VM will be started during system bootup.",
 178         default => 0,
 179     },
 180     autostart => {
 181         optional => 1,
 182         type => 'boolean',
 183         description => "Automatic restart after crash (currently ignored).",
 184         default => 0,
 185     },
 186     hotplug => {
 187         optional => 1,
 188         type => 'string', format => 'pve-hotplug-features',
 189         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 190         default => 'network,disk,usb',
 191     },
 192     reboot => {
 193         optional => 1,
 194         type => 'boolean',
 195         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 196         default => 1,
 197     },
 198     lock => {
 199         optional => 1,
 200         type => 'string',
 201         description => "Lock/unlock the VM.",
 202         enum => [qw(migrate backup snapshot rollback)],
 203     },
 204     cpulimit => {
 205         optional => 1,
 206         type => 'number',
 207         description => "Limit of CPU usage.",
 208         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 209         minimum => 0,
 210         maximum => 128,
 211         default => 0,
 212     },
 213     cpuunits => {
 214         optional => 1,
 215         type => 'integer',
 216         description => "CPU weight for a VM.",
 217         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 218         minimum => 2,
 219         maximum => 262144,
 220         default => 1024,
 221     },
 222     memory => {
 223         optional => 1,
 224         type => 'integer',
 225         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 226         minimum => 16,
 227         default => 512,
 228     },
 229     balloon => {
 230         optional => 1,
 231         type => 'integer',
 232         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 233         minimum => 0,
 234     },
 235     shares => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 239         minimum => 0,
 240         maximum => 50000,
 241         default => 1000,
 242     },
 243     keyboard => {
 244         optional => 1,
 245         type => 'string',
 246         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 247         enum => PVE::Tools::kvmkeymaplist(),
 248         default => 'en-us',
 249     },
 250     name => {
 251         optional => 1,
 252         type => 'string', format => 'dns-name',
 253         description => "Set a name for the VM. Only used on the configuration web interface.",
 254     },
 255     scsihw => {
 256         optional => 1,
 257         type => 'string',
 258         description => "SCSI controller model",
 259         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 260         default => 'lsi',
 261     },
 262     description => {
 263         optional => 1,
 264         type => 'string',
 265         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 266     },
 267     ostype => {
 268         optional => 1,
 269         type => 'string',
 270         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 271         description => "Specify guest operating system.",
 272         verbose_description => <<EODESC,
 273 Specify guest operating system. This is used to enable special
 274 optimization/features for specific operating systems:
 275
 276 [horizontal]
 277 other;; unspecified OS
 278 wxp;; Microsoft Windows XP
 279 w2k;; Microsoft Windows 2000
 280 w2k3;; Microsoft Windows 2003
 281 w2k8;; Microsoft Windows 2008
 282 wvista;; Microsoft Windows Vista
 283 win7;; Microsoft Windows 7
 284 win8;; Microsoft Windows 8/2012
 285 l24;; Linux 2.4 Kernel
 286 l26;; Linux 2.6/3.X Kernel
 287 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 288 EODESC
 289     },
 290     boot => {
 291         optional => 1,
 292         type => 'string',
 293         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 294         pattern => '[acdn]{1,4}',
 295         default => 'cdn',
 296     },
 297     bootdisk => {
 298         optional => 1,
 299         type => 'string', format => 'pve-qm-bootdisk',
 300         description => "Enable booting from specified disk.",
 301         pattern => '(ide|sata|scsi|virtio)\d+',
 302     },
 303     smp => {
 304         optional => 1,
 305         type => 'integer',
 306         description => "The number of CPUs. Please use option -sockets instead.",
 307         minimum => 1,
 308         default => 1,
 309     },
 310     sockets => {
 311         optional => 1,
 312         type => 'integer',
 313         description => "The number of CPU sockets.",
 314         minimum => 1,
 315         default => 1,
 316     },
 317     cores => {
 318         optional => 1,
 319         type => 'integer',
 320         description => "The number of cores per socket.",
 321         minimum => 1,
 322         default => 1,
 323     },
 324     numa => {
 325         optional => 1,
 326         type => 'boolean',
 327         description => "Enable/disable NUMA.",
 328         default => 0,
 329     },
 330     hugepages => {
 331         optional => 1,
 332         type => 'string',
 333         description => "Enable/disable hugepages memory.",
 334         enum => [qw(any 2 1024)],
 335     },
 336     vcpus => {
 337         optional => 1,
 338         type => 'integer',
 339         description => "Number of hotplugged vcpus.",
 340         minimum => 1,
 341         default => 0,
 342     },
 343     acpi => {
 344         optional => 1,
 345         type => 'boolean',
 346         description => "Enable/disable ACPI.",
 347         default => 1,
 348     },
 349     agent => {
 350         optional => 1,
 351         type => 'boolean',
 352         description => "Enable/disable Qemu GuestAgent.",
 353         default => 0,
 354     },
 355     kvm => {
 356         optional => 1,
 357         type => 'boolean',
 358         description => "Enable/disable KVM hardware virtualization.",
 359         default => 1,
 360     },
 361     tdf => {
 362         optional => 1,
 363         type => 'boolean',
 364         description => "Enable/disable time drift fix.",
 365         default => 0,
 366     },
 367     localtime => {
 368         optional => 1,
 369         type => 'boolean',
 370         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 371     },
 372     freeze => {
 373         optional => 1,
 374         type => 'boolean',
 375         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 376     },
 377     vga => {
 378         optional => 1,
 379         type => 'string',
 380         description => "Select the VGA type.",
 381         verbose_description => "Select the VGA type. If you want to use high resolution" .
 382             " modes (>= 1280x1024x16) then you should use the options " .
 383             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 384             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 385             "display sever. For win* OS you can select how many independent " .
 386             "displays you want, Linux guests can add displays them self. " .
 387             "You can also run without any graphic card, using a serial device" .
 388             " as terminal.",
 389         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 390     },
 391     watchdog => {
 392         optional => 1,
 393         type => 'string', format => 'pve-qm-watchdog',
 394         description => "Create a virtual hardware watchdog device.",
 395         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 396             " (by a guest action), the watchdog must be periodically polled " .
 397             "by an agent inside the guest or else the watchdog will reset " .
 398             "the guest (or execute the respective action specified)",
 399     },
 400     startdate => {
 401         optional => 1,
 402         type => 'string',
 403         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 404         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 405         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 406         default => 'now',
 407     },
 408     startup =>  get_standard_option('pve-startup-order'),
 409     template => {
 410         optional => 1,
 411         type => 'boolean',
 412         description => "Enable/disable Template.",
 413         default => 0,
 414     },
 415     args => {
 416         optional => 1,
 417         type => 'string',
 418         description => "Arbitrary arguments passed to kvm.",
 419         verbose_description => <<EODESCR,
 420 Arbitrary arguments passed to kvm, for example:
 421
 422 args: -no-reboot -no-hpet
 423
 424 NOTE: this option is for experts only.
 425 EODESCR
 426     },
 427     tablet => {
 428         optional => 1,
 429         type => 'boolean',
 430         default => 1,
 431         description => "Enable/disable the USB tablet device.",
 432         verbose_description => "Enable/disable the USB tablet device. This device is " .
 433             "usually needed to allow absolute mouse positioning with VNC. " .
 434             "Else the mouse runs out of sync with normal VNC clients. " .
 435             "If you're running lots of console-only guests on one host, " .
 436             "you may consider disabling this to save some context switches. " .
 437             "This is turned off by default if you use spice (-vga=qxl).",
 438     },
 439     migrate_speed => {
 440         optional => 1,
 441         type => 'integer',
 442         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 443         minimum => 0,
 444         default => 0,
 445     },
 446     migrate_downtime => {
 447         optional => 1,
 448         type => 'number',
 449         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 450         minimum => 0,
 451         default => 0.1,
 452     },
 453     cdrom => {
 454         optional => 1,
 455         type => 'string', format => 'pve-qm-ide',
 456         typetext => '<volume>',
 457         description => "This is an alias for option -ide2",
 458     },
 459     cpu => {
 460         optional => 1,
 461         description => "Emulated CPU type.",
 462         type => 'string',
 463         format => $cpu_fmt,
 464     },
 465     parent => get_standard_option('pve-snapshot-name', {
 466         optional => 1,
 467         description => "Parent snapshot name. This is used internally, and should not be modified.",
 468     }),
 469     snaptime => {
 470         optional => 1,
 471         description => "Timestamp for snapshots.",
 472         type => 'integer',
 473         minimum => 0,
 474     },
 475     vmstate => {
 476         optional => 1,
 477         type => 'string', format => 'pve-volume-id',
 478         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 479     },
 480     vmstatestorage => get_standard_option('pve-storage-id', {
 481         description => "Default storage for VM state volumes/files.",
 482         optional => 1,
 483     }),
 484     machine => {
 485         description => "Specific the Qemu machine type.",
 486         type => 'string',
 487         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 488         maxLength => 40,
 489         optional => 1,
 490     },
 491     smbios1 => {
 492         description => "Specify SMBIOS type 1 fields.",
 493         type => 'string', format => 'pve-qm-smbios1',
 494         maxLength => 256,
 495         optional => 1,
 496     },
 497     protection => {
 498         optional => 1,
 499         type => 'boolean',
 500         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 501         default => 0,
 502     },
 503     bios => {
 504         optional => 1,
 505         type => 'string',
 506         enum => [ qw(seabios ovmf) ],
 507         description => "Select BIOS implementation.",
 508         default => 'seabios',
 509     },
 510 };
 511
 512 # what about other qemu settings ?
 513 #cpu => 'string',
 514 #machine => 'string',
 515 #fda => 'file',
 516 #fdb => 'file',
 517 #mtdblock => 'file',
 518 #sd => 'file',
 519 #pflash => 'file',
 520 #snapshot => 'bool',
 521 #bootp => 'file',
 522 ##tftp => 'dir',
 523 ##smb => 'dir',
 524 #kernel => 'file',
 525 #append => 'string',
 526 #initrd => 'file',
 527 ##soundhw => 'string',
 528
 529 while (my ($k, $v) = each %$confdesc) {
 530     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 531 }
 532
 533 my $MAX_IDE_DISKS = 4;
 534 my $MAX_SCSI_DISKS = 14;
 535 my $MAX_VIRTIO_DISKS = 16;
 536 my $MAX_SATA_DISKS = 6;
 537 my $MAX_USB_DEVICES = 5;
 538 my $MAX_NETS = 32;
 539 my $MAX_UNUSED_DISKS = 8;
 540 my $MAX_HOSTPCI_DEVICES = 4;
 541 my $MAX_SERIAL_PORTS = 4;
 542 my $MAX_PARALLEL_PORTS = 3;
 543 my $MAX_NUMA = 8;
 544
 545 my $numa_fmt = {
 546     cpus => {
 547         type => "string",
 548         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 549         description => "CPUs accessing this NUMA node.",
 550         format_description => "id[-id];...",
 551     },
 552     memory => {
 553         type => "number",
 554         description => "Amount of memory this NUMA node provides.",
 555         optional => 1,
 556     },
 557     hostnodes => {
 558         type => "string",
 559         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 560         description => "Host NUMA nodes to use.",
 561         format_description => "id[-id];...",
 562         optional => 1,
 563     },
 564     policy => {
 565         type => 'string',
 566         enum => [qw(preferred bind interleave)],
 567         description => "NUMA allocation policy.",
 568         optional => 1,
 569     },
 570 };
 571 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 572 my $numadesc = {
 573     optional => 1,
 574     type => 'string', format => $numa_fmt,
 575     description => "NUMA topology.",
 576 };
 577 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 578
 579 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 580     $confdesc->{"numa$i"} = $numadesc;
 581 }
 582
 583 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 584                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 585                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 586 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 587
 588 my $net_fmt_bridge_descr = <<__EOD__;
 589 Bridge to attach the network device to. The Proxmox VE standard bridge
 590 is called 'vmbr0'.
 591
 592 If you do not specify a bridge, we create a kvm user (NATed) network
 593 device, which provides DHCP and DNS services. The following addresses
 594 are used:
 595
 596  10.0.2.2   Gateway
 597  10.0.2.3   DNS Server
 598  10.0.2.4   SMB Server
 599
 600 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 601 __EOD__
 602
 603 my $net_fmt = {
 604     macaddr => {
 605         type => 'string',
 606         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 607         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 608         format_description => "XX:XX:XX:XX:XX:XX",
 609         optional => 1,
 610     },
 611     model => {
 612         type => 'string',
 613         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 614         enum => $nic_model_list,
 615         default_key => 1,
 616     },
 617     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 618     bridge => {
 619         type => 'string',
 620         description => $net_fmt_bridge_descr,
 621         format_description => 'bridge',
 622         optional => 1,
 623     },
 624     queues => {
 625         type => 'integer',
 626         minimum => 0, maximum => 16,
 627         description => 'Number of packet queues to be used on the device.',
 628         optional => 1,
 629     },
 630     rate => {
 631         type => 'number',
 632         minimum => 0,
 633         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 634         optional => 1,
 635     },
 636     tag => {
 637         type => 'integer',
 638         minimum => 1, maximum => 4094,
 639         description => 'VLAN tag to apply to packets on this interface.',
 640         optional => 1,
 641     },
 642     trunks => {
 643         type => 'string',
 644         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 645         description => 'VLAN trunks to pass through this interface.',
 646         format_description => 'vlanid[;vlanid...]',
 647         optional => 1,
 648     },
 649     firewall => {
 650         type => 'boolean',
 651         description => 'Whether this interface should be protected by the firewall.',
 652         optional => 1,
 653     },
 654     link_down => {
 655         type => 'boolean',
 656         description => 'Whether this interface should be disconnected (like pulling the plug).',
 657         optional => 1,
 658     },
 659 };
 660
 661 my $netdesc = {
 662     optional => 1,
 663     type => 'string', format => $net_fmt,
 664     description => "Specify network devices.",
 665 };
 666
 667 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 668
 669 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 670     $confdesc->{"net$i"} = $netdesc;
 671 }
 672
 673 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 674 sub verify_volume_id_or_qm_path {
 675     my ($volid, $noerr) = @_;
 676
 677     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 678         return $volid;
 679     }
 680
 681     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 682     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 683     if ($@) {
 684         return undef if $noerr;
 685         die $@;
 686     }
 687     return $volid;
 688 }
 689
 690 my $drivename_hash;
 691
 692 my %drivedesc_base = (
 693     volume => { alias => 'file' },
 694     file => {
 695         type => 'string',
 696         format => 'pve-volume-id-or-qm-path',
 697         default_key => 1,
 698         format_description => 'volume',
 699         description => "The drive's backing volume.",
 700     },
 701     media => {
 702         type => 'string',
 703         enum => [qw(cdrom disk)],
 704         description => "The drive's media type.",
 705         default => 'disk',
 706         optional => 1
 707     },
 708     cyls => {
 709         type => 'integer',
 710         description => "Force the drive's physical geometry to have a specific cylinder count.",
 711         optional => 1
 712     },
 713     heads => {
 714         type => 'integer',
 715         description => "Force the drive's physical geometry to have a specific head count.",
 716         optional => 1
 717     },
 718     secs => {
 719         type => 'integer',
 720         description => "Force the drive's physical geometry to have a specific sector count.",
 721         optional => 1
 722     },
 723     trans => {
 724         type => 'string',
 725         enum => [qw(none lba auto)],
 726         description => "Force disk geometry bios translation mode.",
 727         optional => 1,
 728     },
 729     snapshot => {
 730         type => 'boolean',
 731         description => "Whether the drive should be included when making snapshots.",
 732         optional => 1,
 733     },
 734     cache => {
 735         type => 'string',
 736         enum => [qw(none writethrough writeback unsafe directsync)],
 737         description => "The drive's cache mode",
 738         optional => 1,
 739     },
 740     format => get_standard_option('pve-qm-image-format'),
 741     size => {
 742         type => 'string',
 743         format => 'disk-size',
 744         format_description => 'DiskSize',
 745         description => "Disk size. This is purely informational and has no effect.",
 746         optional => 1,
 747     },
 748     backup => {
 749         type => 'boolean',
 750         description => "Whether the drive should be included when making backups.",
 751         optional => 1,
 752     },
 753     replicate => {
 754         type => 'boolean',
 755         description => 'Whether the drive should considered for replication jobs.',
 756         optional => 1,
 757         default => 1,
 758     },
 759     rerror => {
 760         type => 'string',
 761         enum => [qw(ignore report stop)],
 762         description => 'Read error action.',
 763         optional => 1,
 764     },
 765     werror => {
 766         type => 'string',
 767         enum => [qw(enospc ignore report stop)],
 768         description => 'Write error action.',
 769         optional => 1,
 770     },
 771     aio => {
 772         type => 'string',
 773         enum => [qw(native threads)],
 774         description => 'AIO type to use.',
 775         optional => 1,
 776     },
 777     discard => {
 778         type => 'string',
 779         enum => [qw(ignore on)],
 780         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 781         optional => 1,
 782     },
 783     detect_zeroes => {
 784         type => 'boolean',
 785         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 786         optional => 1,
 787     },
 788     serial => {
 789         type => 'string',
 790         format => 'urlencoded',
 791         format_description => 'serial',
 792         maxLength => 20*3, # *3 since it's %xx url enoded
 793         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 794         optional => 1,
 795     }
 796 );
 797
 798 my %iothread_fmt = ( iothread => {
 799         type => 'boolean',
 800         description => "Whether to use iothreads for this drive",
 801         optional => 1,
 802 });
 803
 804 my %model_fmt = (
 805     model => {
 806         type => 'string',
 807         format => 'urlencoded',
 808         format_description => 'model',
 809         maxLength => 40*3, # *3 since it's %xx url enoded
 810         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 811         optional => 1,
 812     },
 813 );
 814
 815 my %queues_fmt = (
 816     queues => {
 817         type => 'integer',
 818         description => "Number of queues.",
 819         minimum => 2,
 820         optional => 1
 821     }
 822 );
 823
 824 my %scsiblock_fmt = (
 825     scsiblock => {
 826         type => 'boolean',
 827         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 828         optional => 1,
 829         default => 0,
 830     },
 831 );
 832
 833 my $add_throttle_desc = sub {
 834     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 835     my $d = {
 836         type => $type,
 837         format_description => $unit,
 838         description => "Maximum $what in $longunit.",
 839         optional => 1,
 840     };
 841     $d->{minimum} = $minimum if defined($minimum);
 842     $drivedesc_base{$key} = $d;
 843 };
 844 # throughput: (leaky bucket)
 845 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 846 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 847 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 848 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 849 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 850 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 851 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 852 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 853 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 854
 855 # pools: (pool of IO before throttling starts taking effect)
 856 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 857 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 858 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 859 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 860 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 861 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 862
 863 # burst lengths
 864 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 865 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 866 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 867 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 868 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 869 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 870
 871 # legacy support
 872 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
 873 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
 874 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
 875 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
 876
 877 my $ide_fmt = {
 878     %drivedesc_base,
 879     %model_fmt,
 880 };
 881 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 882
 883 my $idedesc = {
 884     optional => 1,
 885     type => 'string', format => $ide_fmt,
 886     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 887 };
 888 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 889
 890 my $scsi_fmt = {
 891     %drivedesc_base,
 892     %iothread_fmt,
 893     %queues_fmt,
 894     %scsiblock_fmt,
 895 };
 896 my $scsidesc = {
 897     optional => 1,
 898     type => 'string', format => $scsi_fmt,
 899     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 900 };
 901 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 902
 903 my $sata_fmt = {
 904     %drivedesc_base,
 905 };
 906 my $satadesc = {
 907     optional => 1,
 908     type => 'string', format => $sata_fmt,
 909     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 910 };
 911 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 912
 913 my $virtio_fmt = {
 914     %drivedesc_base,
 915     %iothread_fmt,
 916 };
 917 my $virtiodesc = {
 918     optional => 1,
 919     type => 'string', format => $virtio_fmt,
 920     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 921 };
 922 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 923
 924 my $alldrive_fmt = {
 925     %drivedesc_base,
 926     %iothread_fmt,
 927     %model_fmt,
 928     %queues_fmt,
 929     %scsiblock_fmt,
 930 };
 931
 932 my $efidisk_fmt = {
 933     volume => { alias => 'file' },
 934     file => {
 935         type => 'string',
 936         format => 'pve-volume-id-or-qm-path',
 937         default_key => 1,
 938         format_description => 'volume',
 939         description => "The drive's backing volume.",
 940     },
 941     format => get_standard_option('pve-qm-image-format'),
 942     size => {
 943         type => 'string',
 944         format => 'disk-size',
 945         format_description => 'DiskSize',
 946         description => "Disk size. This is purely informational and has no effect.",
 947         optional => 1,
 948     },
 949 };
 950
 951 my $efidisk_desc = {
 952     optional => 1,
 953     type => 'string', format => $efidisk_fmt,
 954     description => "Configure a Disk for storing EFI vars",
 955 };
 956
 957 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 958
 959 my $usb_fmt = {
 960     host => {
 961         default_key => 1,
 962         type => 'string', format => 'pve-qm-usb-device',
 963         format_description => 'HOSTUSBDEVICE|spice',
 964         description => <<EODESCR,
 965 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 966
 967  'bus-port(.port)*' (decimal numbers) or
 968  'vendor_id:product_id' (hexadeciaml numbers) or
 969  'spice'
 970
 971 You can use the 'lsusb -t' command to list existing usb devices.
 972
 973 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 974
 975 The value 'spice' can be used to add a usb redirection devices for spice.
 976 EODESCR
 977     },
 978     usb3 => {
 979         optional => 1,
 980         type => 'boolean',
 981         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 982         default => 0,
 983     },
 984 };
 985
 986 my $usbdesc = {
 987     optional => 1,
 988     type => 'string', format => $usb_fmt,
 989     description => "Configure an USB device (n is 0 to 4).",
 990 };
 991 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 992
 993 # NOTE: the match-groups of this regex are used in parse_hostpci
 994 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 995 my $hostpci_fmt = {
 996     host => {
 997         default_key => 1,
 998         type => 'string',
 999         pattern => qr/$PCIRE(;$PCIRE)*/,
1000         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1001         description => <<EODESCR,
1002 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1003 of PCI virtual functions of the host. HOSTPCIID syntax is:
1004
1005 'bus:dev.func' (hexadecimal numbers)
1006
1007 You can us the 'lspci' command to list existing PCI devices.
1008 EODESCR
1009     },
1010     rombar => {
1011         type => 'boolean',
1012         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1013         optional => 1,
1014         default => 1,
1015     },
1016     romfile => {
1017         type => 'string',
1018         pattern => '[^,;]+',
1019         format_description => 'string',
1020         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1021         optional => 1,
1022     },
1023     pcie => {
1024         type => 'boolean',
1025         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1026         optional => 1,
1027         default => 0,
1028     },
1029     'x-vga' => {
1030         type => 'boolean',
1031         description =>  "Enable vfio-vga device support.",
1032         optional => 1,
1033         default => 0,
1034     },
1035 };
1036 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1037
1038 my $hostpcidesc = {
1039         optional => 1,
1040         type => 'string', format => 'pve-qm-hostpci',
1041         description => "Map host PCI devices into guest.",
1042         verbose_description =>  <<EODESCR,
1043 Map host PCI devices into guest.
1044
1045 NOTE: This option allows direct access to host hardware. So it is no longer
1046 possible to migrate such machines - use with special care.
1047
1048 CAUTION: Experimental! User reported problems with this option.
1049 EODESCR
1050 };
1051 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1052
1053 my $serialdesc = {
1054         optional => 1,
1055         type => 'string',
1056         pattern => '(/dev/.+|socket)',
1057         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1058         verbose_description =>  <<EODESCR,
1059 Create a serial device inside the VM (n is 0 to 3), and pass through a
1060 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1061 host side (use 'qm terminal' to open a terminal connection).
1062
1063 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1064
1065 CAUTION: Experimental! User reported problems with this option.
1066 EODESCR
1067 };
1068
1069 my $paralleldesc= {
1070         optional => 1,
1071         type => 'string',
1072         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1073         description =>  "Map host parallel devices (n is 0 to 2).",
1074         verbose_description =>  <<EODESCR,
1075 Map host parallel devices (n is 0 to 2).
1076
1077 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1078
1079 CAUTION: Experimental! User reported problems with this option.
1080 EODESCR
1081 };
1082
1083 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1084     $confdesc->{"parallel$i"} = $paralleldesc;
1085 }
1086
1087 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1088     $confdesc->{"serial$i"} = $serialdesc;
1089 }
1090
1091 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1092     $confdesc->{"hostpci$i"} = $hostpcidesc;
1093 }
1094
1095 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1096     $drivename_hash->{"ide$i"} = 1;
1097     $confdesc->{"ide$i"} = $idedesc;
1098 }
1099
1100 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1101     $drivename_hash->{"sata$i"} = 1;
1102     $confdesc->{"sata$i"} = $satadesc;
1103 }
1104
1105 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1106     $drivename_hash->{"scsi$i"} = 1;
1107     $confdesc->{"scsi$i"} = $scsidesc ;
1108 }
1109
1110 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1111     $drivename_hash->{"virtio$i"} = 1;
1112     $confdesc->{"virtio$i"} = $virtiodesc;
1113 }
1114
1115 $drivename_hash->{efidisk0} = 1;
1116 $confdesc->{efidisk0} = $efidisk_desc;
1117
1118 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1119     $confdesc->{"usb$i"} = $usbdesc;
1120 }
1121
1122 my $unuseddesc = {
1123     optional => 1,
1124     type => 'string', format => 'pve-volume-id',
1125     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1126 };
1127
1128 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1129     $confdesc->{"unused$i"} = $unuseddesc;
1130 }
1131
1132 my $kvm_api_version = 0;
1133
1134 sub kvm_version {
1135
1136     return $kvm_api_version if $kvm_api_version;
1137
1138     my $fh = IO::File->new("</dev/kvm") ||
1139         return 0;
1140
1141     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1142         $kvm_api_version = $v;
1143     }
1144
1145     $fh->close();
1146
1147     return  $kvm_api_version;
1148 }
1149
1150 my $kvm_user_version;
1151
1152 sub kvm_user_version {
1153
1154     return $kvm_user_version if $kvm_user_version;
1155
1156     $kvm_user_version = 'unknown';
1157
1158     my $code = sub {
1159         my $line = shift;
1160         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1161             $kvm_user_version = $2;
1162         }
1163     };
1164
1165     eval { run_command("kvm -version", outfunc => $code); };
1166     warn $@ if $@;
1167
1168     return $kvm_user_version;
1169
1170 }
1171
1172 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1173
1174 sub valid_drive_names {
1175     # order is important - used to autoselect boot disk
1176     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1177             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1178             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1179             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1180             'efidisk0');
1181 }
1182
1183 sub is_valid_drivename {
1184     my $dev = shift;
1185
1186     return defined($drivename_hash->{$dev});
1187 }
1188
1189 sub option_exists {
1190     my $key = shift;
1191     return defined($confdesc->{$key});
1192 }
1193
1194 sub nic_models {
1195     return $nic_model_list;
1196 }
1197
1198 sub os_list_description {
1199
1200     return {
1201         other => 'Other',
1202         wxp => 'Windows XP',
1203         w2k => 'Windows 2000',
1204         w2k3 =>, 'Windows 2003',
1205         w2k8 => 'Windows 2008',
1206         wvista => 'Windows Vista',
1207         win7 => 'Windows 7',
1208         win8 => 'Windows 8/2012',
1209         win10 => 'Windows 10/2016',
1210         l24 => 'Linux 2.4',
1211         l26 => 'Linux 2.6',
1212     };
1213 }
1214
1215 my $cdrom_path;
1216
1217 sub get_cdrom_path {
1218
1219     return  $cdrom_path if $cdrom_path;
1220
1221     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1222     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1223     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1224 }
1225
1226 sub get_iso_path {
1227     my ($storecfg, $vmid, $cdrom) = @_;
1228
1229     if ($cdrom eq 'cdrom') {
1230         return get_cdrom_path();
1231     } elsif ($cdrom eq 'none') {
1232         return '';
1233     } elsif ($cdrom =~ m|^/|) {
1234         return $cdrom;
1235     } else {
1236         return PVE::Storage::path($storecfg, $cdrom);
1237     }
1238 }
1239
1240 # try to convert old style file names to volume IDs
1241 sub filename_to_volume_id {
1242     my ($vmid, $file, $media) = @_;
1243
1244     if (!($file eq 'none' || $file eq 'cdrom' ||
1245           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1246
1247         return undef if $file =~ m|/|;
1248
1249         if ($media && $media eq 'cdrom') {
1250             $file = "local:iso/$file";
1251         } else {
1252             $file = "local:$vmid/$file";
1253         }
1254     }
1255
1256     return $file;
1257 }
1258
1259 sub verify_media_type {
1260     my ($opt, $vtype, $media) = @_;
1261
1262     return if !$media;
1263
1264     my $etype;
1265     if ($media eq 'disk') {
1266         $etype = 'images';
1267     } elsif ($media eq 'cdrom') {
1268         $etype = 'iso';
1269     } else {
1270         die "internal error";
1271     }
1272
1273     return if ($vtype eq $etype);
1274
1275     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1276 }
1277
1278 sub cleanup_drive_path {
1279     my ($opt, $storecfg, $drive) = @_;
1280
1281     # try to convert filesystem paths to volume IDs
1282
1283     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1284         ($drive->{file} !~ m|^/dev/.+|) &&
1285         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1286         ($drive->{file} !~ m/^\d+$/)) {
1287         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1288         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1289         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1290         verify_media_type($opt, $vtype, $drive->{media});
1291         $drive->{file} = $volid;
1292     }
1293
1294     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1295 }
1296
1297 sub parse_hotplug_features {
1298     my ($data) = @_;
1299
1300     my $res = {};
1301
1302     return $res if $data eq '0';
1303
1304     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1305
1306     foreach my $feature (PVE::Tools::split_list($data)) {
1307         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1308             $res->{$1} = 1;
1309         } else {
1310             die "invalid hotplug feature '$feature'\n";
1311         }
1312     }
1313     return $res;
1314 }
1315
1316 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1317 sub pve_verify_hotplug_features {
1318     my ($value, $noerr) = @_;
1319
1320     return $value if parse_hotplug_features($value);
1321
1322     return undef if $noerr;
1323
1324     die "unable to parse hotplug option\n";
1325 }
1326
1327 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1328 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1329 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1330 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1331 #        [,iothread=on][,serial=serial][,model=model]
1332
1333 sub parse_drive {
1334     my ($key, $data) = @_;
1335
1336     my ($interface, $index);
1337
1338     if ($key =~ m/^([^\d]+)(\d+)$/) {
1339         $interface = $1;
1340         $index = $2;
1341     } else {
1342         return undef;
1343     }
1344
1345     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1346                                      : $confdesc->{$key}->{format};
1347     if (!$desc) {
1348         warn "invalid drive key: $key\n";
1349         return undef;
1350     }
1351     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1352     return undef if !$res;
1353     $res->{interface} = $interface;
1354     $res->{index} = $index;
1355
1356     my $error = 0;
1357     foreach my $opt (qw(bps bps_rd bps_wr)) {
1358         if (my $bps = defined(delete $res->{$opt})) {
1359             if (defined($res->{"m$opt"})) {
1360                 warn "both $opt and m$opt specified\n";
1361                 ++$error;
1362                 next;
1363             }
1364             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1365         }
1366     }
1367
1368     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1369     for my $requirement (
1370         [mbps_max => 'mbps'],
1371         [mbps_rd_max => 'mbps_rd'],
1372         [mbps_wr_max => 'mbps_wr'],
1373         [miops_max => 'miops'],
1374         [miops_rd_max => 'miops_rd'],
1375         [miops_wr_max => 'miops_wr'],
1376         [bps_max_length => 'mbps_max'],
1377         [bps_rd_max_length => 'mbps_rd_max'],
1378         [bps_wr_max_length => 'mbps_wr_max'],
1379         [iops_max_length => 'iops_max'],
1380         [iops_rd_max_length => 'iops_rd_max'],
1381         [iops_wr_max_length => 'iops_wr_max']) {
1382         my ($option, $requires) = @$requirement;
1383         if ($res->{$option} && !$res->{$requires}) {
1384             warn "$option requires $requires\n";
1385             ++$error;
1386         }
1387     }
1388
1389     return undef if $error;
1390
1391     return undef if $res->{mbps_rd} && $res->{mbps};
1392     return undef if $res->{mbps_wr} && $res->{mbps};
1393     return undef if $res->{iops_rd} && $res->{iops};
1394     return undef if $res->{iops_wr} && $res->{iops};
1395
1396     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1397         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1398         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1399         return undef if $res->{interface} eq 'virtio';
1400     }
1401
1402     if (my $size = $res->{size}) {
1403         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1404     }
1405
1406     return $res;
1407 }
1408
1409 sub print_drive {
1410     my ($vmid, $drive) = @_;
1411     my $data = { %$drive };
1412     delete $data->{$_} for qw(index interface);
1413     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1414 }
1415
1416 sub scsi_inquiry {
1417     my($fh, $noerr) = @_;
1418
1419     my $SG_IO = 0x2285;
1420     my $SG_GET_VERSION_NUM = 0x2282;
1421
1422     my $versionbuf = "\x00" x 8;
1423     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1424     if (!$ret) {
1425         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1426         return undef;
1427     }
1428     my $version = unpack("I", $versionbuf);
1429     if ($version < 30000) {
1430         die "scsi generic interface too old\n"  if !$noerr;
1431         return undef;
1432     }
1433
1434     my $buf = "\x00" x 36;
1435     my $sensebuf = "\x00" x 8;
1436     my $cmd = pack("C x3 C x1", 0x12, 36);
1437
1438     # see /usr/include/scsi/sg.h
1439     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1440
1441     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1442                       length($sensebuf), 0, length($buf), $buf,
1443                       $cmd, $sensebuf, 6000);
1444
1445     $ret = ioctl($fh, $SG_IO, $packet);
1446     if (!$ret) {
1447         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1448         return undef;
1449     }
1450
1451     my @res = unpack($sg_io_hdr_t, $packet);
1452     if ($res[17] || $res[18]) {
1453         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1454         return undef;
1455     }
1456
1457     my $res = {};
1458     (my $byte0, my $byte1, $res->{vendor},
1459      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1460
1461     $res->{removable} = $byte1 & 128 ? 1 : 0;
1462     $res->{type} = $byte0 & 31;
1463
1464     return $res;
1465 }
1466
1467 sub path_is_scsi {
1468     my ($path) = @_;
1469
1470     my $fh = IO::File->new("+<$path") || return undef;
1471     my $res = scsi_inquiry($fh, 1);
1472     close($fh);
1473
1474     return $res;
1475 }
1476
1477 sub machine_type_is_q35 {
1478     my ($conf) = @_;
1479
1480     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1481 }
1482
1483 sub print_tabletdevice_full {
1484     my ($conf) = @_;
1485
1486     my $q35 = machine_type_is_q35($conf);
1487
1488     # we use uhci for old VMs because tablet driver was buggy in older qemu
1489     my $usbbus = $q35 ? "ehci" : "uhci";
1490
1491     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1492 }
1493
1494 sub print_drivedevice_full {
1495     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1496
1497     my $device = '';
1498     my $maxdev = 0;
1499
1500     if ($drive->{interface} eq 'virtio') {
1501         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1502         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1503         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1504     } elsif ($drive->{interface} eq 'scsi') {
1505
1506         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1507         my $unit = $drive->{index} % $maxdev;
1508         my $devicetype = 'hd';
1509         my $path = '';
1510         if (drive_is_cdrom($drive)) {
1511             $devicetype = 'cd';
1512         } else {
1513             if ($drive->{file} =~ m|^/|) {
1514                 $path = $drive->{file};
1515                 if (my $info = path_is_scsi($path)) {
1516                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1517                         $devicetype = 'block';
1518                     } elsif ($info->{type} == 1) { # tape
1519                         $devicetype = 'generic';
1520                     }
1521                 }
1522             } else {
1523                  $path = PVE::Storage::path($storecfg, $drive->{file});
1524             }
1525
1526             if($path =~ m/^iscsi\:\/\//){
1527                 $devicetype = 'generic';
1528             }
1529         }
1530
1531         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1532             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1533         } else {
1534             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1535         }
1536
1537     } elsif ($drive->{interface} eq 'ide'){
1538         $maxdev = 2;
1539         my $controller = int($drive->{index} / $maxdev);
1540         my $unit = $drive->{index} % $maxdev;
1541         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1542
1543         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1544         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1545             $model = URI::Escape::uri_unescape($model);
1546             $device .= ",model=$model";
1547         }
1548     } elsif ($drive->{interface} eq 'sata'){
1549         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1550         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1551         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1552     } elsif ($drive->{interface} eq 'usb') {
1553         die "implement me";
1554         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1555     } else {
1556         die "unsupported interface type";
1557     }
1558
1559     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1560
1561     return $device;
1562 }
1563
1564 sub get_initiator_name {
1565     my $initiator;
1566
1567     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1568     while (defined(my $line = <$fh>)) {
1569         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1570         $initiator = $1;
1571         last;
1572     }
1573     $fh->close();
1574
1575     return $initiator;
1576 }
1577
1578 sub print_drive_full {
1579     my ($storecfg, $vmid, $drive) = @_;
1580
1581     my $path;
1582     my $volid = $drive->{file};
1583     my $format;
1584
1585     if (drive_is_cdrom($drive)) {
1586         $path = get_iso_path($storecfg, $vmid, $volid);
1587     } else {
1588         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1589         if ($storeid) {
1590             $path = PVE::Storage::path($storecfg, $volid);
1591             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1592             $format = qemu_img_format($scfg, $volname);
1593         } else {
1594             $path = $volid;
1595             $format = "raw";
1596         }
1597    }
1598
1599     my $opts = '';
1600     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard);
1601     foreach my $o (@qemu_drive_options) {
1602         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1603     }
1604     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1605         my ($dir, $qmpname) = @$type;
1606         if (my $v = $drive->{"mbps$dir"}) {
1607             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1608         }
1609         if (my $v = $drive->{"mbps${dir}_max"}) {
1610             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1611         }
1612         if (my $v = $drive->{"bps${dir}_max_length"}) {
1613             $opts .= ",throttling.bps$qmpname-max-length=$v";
1614         }
1615         if (my $v = $drive->{"iops${dir}"}) {
1616             $opts .= ",throttling.iops$qmpname=$v";
1617         }
1618         if (my $v = $drive->{"iops${dir}_max"}) {
1619             $opts .= ",throttling.iops$qmpname-max=$v";
1620         }
1621         if (my $v = $drive->{"iops${dir}_max_length"}) {
1622             $opts .= ",throttling.iops$qmpname-max-length=$v";
1623         }
1624     }
1625
1626     if (my $serial = $drive->{serial}) {
1627         $serial = URI::Escape::uri_unescape($serial);
1628         $opts .= ",serial=$serial";
1629     }
1630
1631     $opts .= ",format=$format" if $format && !$drive->{format};
1632
1633     my $cache_direct = 0;
1634
1635     if (my $cache = $drive->{cache}) {
1636         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1637     } elsif (!drive_is_cdrom($drive)) {
1638         $opts .= ",cache=none";
1639         $cache_direct = 1;
1640     }
1641
1642     # aio native works only with O_DIRECT
1643     if (!$drive->{aio}) {
1644         if($cache_direct) {
1645             $opts .= ",aio=native";
1646         } else {
1647             $opts .= ",aio=threads";
1648         }
1649     }
1650
1651     if (!drive_is_cdrom($drive)) {
1652         my $detectzeroes;
1653         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1654             $detectzeroes = 'off';
1655         } elsif ($drive->{discard}) {
1656             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1657         } else {
1658             # This used to be our default with discard not being specified:
1659             $detectzeroes = 'on';
1660         }
1661         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1662     }
1663
1664     my $pathinfo = $path ? "file=$path," : '';
1665
1666     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1667 }
1668
1669 sub print_netdevice_full {
1670     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1671
1672     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1673
1674     my $device = $net->{model};
1675     if ($net->{model} eq 'virtio') {
1676          $device = 'virtio-net-pci';
1677      };
1678
1679     my $pciaddr = print_pci_addr("$netid", $bridges);
1680     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1681     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1682         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1683         my $vectors = $net->{queues} * 2 + 2;
1684         $tmpstr .= ",vectors=$vectors,mq=on";
1685     }
1686     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1687
1688     if ($use_old_bios_files) {
1689         my $romfile;
1690         if ($device eq 'virtio-net-pci') {
1691             $romfile = 'pxe-virtio.rom';
1692         } elsif ($device eq 'e1000') {
1693             $romfile = 'pxe-e1000.rom';
1694         } elsif ($device eq 'ne2k') {
1695             $romfile = 'pxe-ne2k_pci.rom';
1696         } elsif ($device eq 'pcnet') {
1697             $romfile = 'pxe-pcnet.rom';
1698         } elsif ($device eq 'rtl8139') {
1699             $romfile = 'pxe-rtl8139.rom';
1700         }
1701         $tmpstr .= ",romfile=$romfile" if $romfile;
1702     }
1703
1704     return $tmpstr;
1705 }
1706
1707 sub print_netdev_full {
1708     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1709
1710     my $i = '';
1711     if ($netid =~ m/^net(\d+)$/) {
1712         $i = int($1);
1713     }
1714
1715     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1716
1717     my $ifname = "tap${vmid}i$i";
1718
1719     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1720     die "interface name '$ifname' is too long (max 15 character)\n"
1721         if length($ifname) >= 16;
1722
1723     my $vhostparam = '';
1724     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1725
1726     my $vmname = $conf->{name} || "vm$vmid";
1727
1728     my $netdev = "";
1729     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1730
1731     if ($net->{bridge}) {
1732         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1733     } else {
1734         $netdev = "type=user,id=$netid,hostname=$vmname";
1735     }
1736
1737     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1738
1739     return $netdev;
1740 }
1741
1742
1743 sub print_cpu_device {
1744     my ($conf, $id) = @_;
1745
1746     my $kvm = $conf->{kvm} // 1;
1747     my $cpu = $kvm ? "kvm64" : "qemu64";
1748     if (my $cputype = $conf->{cpu}) {
1749         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1750             or die "Cannot parse cpu description: $cputype\n";
1751         $cpu = $cpuconf->{cputype};
1752     }
1753
1754     my $cores = $conf->{cores} || 1;
1755
1756     my $current_core = ($id - 1) % $cores;
1757     my $current_socket = int(($id - 1 - $current_core)/$cores);
1758
1759     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1760 }
1761
1762 sub drive_is_cdrom {
1763     my ($drive) = @_;
1764
1765     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1766
1767 }
1768
1769 sub parse_number_sets {
1770     my ($set) = @_;
1771     my $res = [];
1772     foreach my $part (split(/;/, $set)) {
1773         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1774             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1775             push @$res, [ $1, $2 ];
1776         } else {
1777             die "invalid range: $part\n";
1778         }
1779     }
1780     return $res;
1781 }
1782
1783 sub parse_numa {
1784     my ($data) = @_;
1785
1786     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1787     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1788     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1789     return $res;
1790 }
1791
1792 sub parse_hostpci {
1793     my ($value) = @_;
1794
1795     return undef if !$value;
1796
1797     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1798
1799     my @idlist = split(/;/, $res->{host});
1800     delete $res->{host};
1801     foreach my $id (@idlist) {
1802         if ($id =~ /^$PCIRE$/) {
1803             if (defined($2)) {
1804                 push @{$res->{pciid}}, { id => $1, function => $2 };
1805             } else {
1806                 my $pcidevices = lspci($1);
1807                 $res->{pciid} = $pcidevices->{$1};
1808             }
1809         } else {
1810             # should have been caught by parse_property_string already
1811             die "failed to parse PCI id: $id\n";
1812         }
1813     }
1814     return $res;
1815 }
1816
1817 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1818 sub parse_net {
1819     my ($data) = @_;
1820
1821     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1822     if ($@) {
1823         warn $@;
1824         return undef;
1825     }
1826     if (!defined($res->{macaddr})) {
1827         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1828         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1829     }
1830     return $res;
1831 }
1832
1833 sub print_net {
1834     my $net = shift;
1835
1836     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1837 }
1838
1839 sub add_random_macs {
1840     my ($settings) = @_;
1841
1842     foreach my $opt (keys %$settings) {
1843         next if $opt !~ m/^net(\d+)$/;
1844         my $net = parse_net($settings->{$opt});
1845         next if !$net;
1846         $settings->{$opt} = print_net($net);
1847     }
1848 }
1849
1850 sub vm_is_volid_owner {
1851     my ($storecfg, $vmid, $volid) = @_;
1852
1853     if ($volid !~  m|^/|) {
1854         my ($path, $owner);
1855         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1856         if ($owner && ($owner == $vmid)) {
1857             return 1;
1858         }
1859     }
1860
1861     return undef;
1862 }
1863
1864 sub split_flagged_list {
1865     my $text = shift || '';
1866     $text =~ s/[,;]/ /g;
1867     $text =~ s/^\s+//;
1868     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1869 }
1870
1871 sub join_flagged_list {
1872     my ($how, $lst) = @_;
1873     join $how, map { $lst->{$_} . $_ } keys %$lst;
1874 }
1875
1876 sub vmconfig_delete_pending_option {
1877     my ($conf, $key, $force) = @_;
1878
1879     delete $conf->{pending}->{$key};
1880     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1881     $pending_delete_hash->{$key} = $force ? '!' : '';
1882     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1883 }
1884
1885 sub vmconfig_undelete_pending_option {
1886     my ($conf, $key) = @_;
1887
1888     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1889     delete $pending_delete_hash->{$key};
1890
1891     if (%$pending_delete_hash) {
1892         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1893     } else {
1894         delete $conf->{pending}->{delete};
1895     }
1896 }
1897
1898 sub vmconfig_register_unused_drive {
1899     my ($storecfg, $vmid, $conf, $drive) = @_;
1900
1901     if (!drive_is_cdrom($drive)) {
1902         my $volid = $drive->{file};
1903         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1904             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1905         }
1906     }
1907 }
1908
1909 sub vmconfig_cleanup_pending {
1910     my ($conf) = @_;
1911
1912     # remove pending changes when nothing changed
1913     my $changes;
1914     foreach my $opt (keys %{$conf->{pending}}) {
1915         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1916             $changes = 1;
1917             delete $conf->{pending}->{$opt};
1918         }
1919     }
1920
1921     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1922     my $pending_delete_hash = {};
1923     while (my ($opt, $force) = each %$current_delete_hash) {
1924         if (defined($conf->{$opt})) {
1925             $pending_delete_hash->{$opt} = $force;
1926         } else {
1927             $changes = 1;
1928         }
1929     }
1930
1931     if (%$pending_delete_hash) {
1932         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1933     } else {
1934         delete $conf->{pending}->{delete};
1935     }
1936
1937     return $changes;
1938 }
1939
1940 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1941 my $smbios1_fmt = {
1942     uuid => {
1943         type => 'string',
1944         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1945         format_description => 'UUID',
1946         description => "Set SMBIOS1 UUID.",
1947         optional => 1,
1948     },
1949     version => {
1950         type => 'string',
1951         pattern => '\S+',
1952         format_description => 'string',
1953         description => "Set SMBIOS1 version.",
1954         optional => 1,
1955     },
1956     serial => {
1957         type => 'string',
1958         pattern => '\S+',
1959         format_description => 'string',
1960         description => "Set SMBIOS1 serial number.",
1961         optional => 1,
1962     },
1963     manufacturer => {
1964         type => 'string',
1965         pattern => '\S+',
1966         format_description => 'string',
1967         description => "Set SMBIOS1 manufacturer.",
1968         optional => 1,
1969     },
1970     product => {
1971         type => 'string',
1972         pattern => '\S+',
1973         format_description => 'string',
1974         description => "Set SMBIOS1 product ID.",
1975         optional => 1,
1976     },
1977     sku => {
1978         type => 'string',
1979         pattern => '\S+',
1980         format_description => 'string',
1981         description => "Set SMBIOS1 SKU string.",
1982         optional => 1,
1983     },
1984     family => {
1985         type => 'string',
1986         pattern => '\S+',
1987         format_description => 'string',
1988         description => "Set SMBIOS1 family string.",
1989         optional => 1,
1990     },
1991 };
1992
1993 sub parse_smbios1 {
1994     my ($data) = @_;
1995
1996     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1997     warn $@ if $@;
1998     return $res;
1999 }
2000
2001 sub print_smbios1 {
2002     my ($smbios1) = @_;
2003     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2004 }
2005
2006 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2007
2008 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2009 sub verify_bootdisk {
2010     my ($value, $noerr) = @_;
2011
2012     return $value if is_valid_drivename($value);
2013
2014     return undef if $noerr;
2015
2016     die "invalid boot disk '$value'\n";
2017 }
2018
2019 sub parse_watchdog {
2020     my ($value) = @_;
2021
2022     return undef if !$value;
2023
2024     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2025     warn $@ if $@;
2026     return $res;
2027 }
2028
2029 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2030 sub verify_usb_device {
2031     my ($value, $noerr) = @_;
2032
2033     return $value if parse_usb_device($value);
2034
2035     return undef if $noerr;
2036
2037     die "unable to parse usb device\n";
2038 }
2039
2040 # add JSON properties for create and set function
2041 sub json_config_properties {
2042     my $prop = shift;
2043
2044     foreach my $opt (keys %$confdesc) {
2045         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2046         $prop->{$opt} = $confdesc->{$opt};
2047     }
2048
2049     return $prop;
2050 }
2051
2052 sub check_type {
2053     my ($key, $value) = @_;
2054
2055     die "unknown setting '$key'\n" if !$confdesc->{$key};
2056
2057     my $type = $confdesc->{$key}->{type};
2058
2059     if (!defined($value)) {
2060         die "got undefined value\n";
2061     }
2062
2063     if ($value =~ m/[\n\r]/) {
2064         die "property contains a line feed\n";
2065     }
2066
2067     if ($type eq 'boolean') {
2068         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2069         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2070         die "type check ('boolean') failed - got '$value'\n";
2071     } elsif ($type eq 'integer') {
2072         return int($1) if $value =~ m/^(\d+)$/;
2073         die "type check ('integer') failed - got '$value'\n";
2074     } elsif ($type eq 'number') {
2075         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2076         die "type check ('number') failed - got '$value'\n";
2077     } elsif ($type eq 'string') {
2078         if (my $fmt = $confdesc->{$key}->{format}) {
2079             PVE::JSONSchema::check_format($fmt, $value);
2080             return $value;
2081         }
2082         $value =~ s/^\"(.*)\"$/$1/;
2083         return $value;
2084     } else {
2085         die "internal error"
2086     }
2087 }
2088
2089 sub check_iommu_support{
2090     #fixme : need to check IOMMU support
2091     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2092
2093     my $iommu=1;
2094     return $iommu;
2095
2096 }
2097
2098 sub touch_config {
2099     my ($vmid) = @_;
2100
2101     my $conf = PVE::QemuConfig->config_file($vmid);
2102     utime undef, undef, $conf;
2103 }
2104
2105 sub destroy_vm {
2106     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2107
2108     my $conffile = PVE::QemuConfig->config_file($vmid);
2109
2110     my $conf = PVE::QemuConfig->load_config($vmid);
2111
2112     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2113
2114     if ($conf->{template}) {
2115         # check if any base image is still used by a linked clone
2116         foreach_drive($conf, sub {
2117                 my ($ds, $drive) = @_;
2118
2119                 return if drive_is_cdrom($drive);
2120
2121                 my $volid = $drive->{file};
2122
2123                 return if !$volid || $volid =~ m|^/|;
2124
2125                 die "base volume '$volid' is still in use by linked cloned\n"
2126                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2127
2128         });
2129     }
2130
2131     # only remove disks owned by this VM
2132     foreach_drive($conf, sub {
2133         my ($ds, $drive) = @_;
2134
2135         return if drive_is_cdrom($drive);
2136
2137         my $volid = $drive->{file};
2138
2139         return if !$volid || $volid =~ m|^/|;
2140
2141         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2142         return if !$path || !$owner || ($owner != $vmid);
2143
2144         eval {
2145             PVE::Storage::vdisk_free($storecfg, $volid);
2146         };
2147         warn "Could not remove disk '$volid', check manually: $@" if $@;
2148
2149     });
2150
2151     if ($keep_empty_config) {
2152         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2153     } else {
2154         unlink $conffile;
2155     }
2156
2157     # also remove unused disk
2158     eval {
2159         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2160
2161         eval {
2162             PVE::Storage::foreach_volid($dl, sub {
2163                 my ($volid, $sid, $volname, $d) = @_;
2164                 PVE::Storage::vdisk_free($storecfg, $volid);
2165             });
2166         };
2167         warn $@ if $@;
2168
2169     };
2170     warn $@ if $@;
2171 }
2172
2173 sub parse_vm_config {
2174     my ($filename, $raw) = @_;
2175
2176     return undef if !defined($raw);
2177
2178     my $res = {
2179         digest => Digest::SHA::sha1_hex($raw),
2180         snapshots => {},
2181         pending => {},
2182     };
2183
2184     $filename =~ m|/qemu-server/(\d+)\.conf$|
2185         || die "got strange filename '$filename'";
2186
2187     my $vmid = $1;
2188
2189     my $conf = $res;
2190     my $descr;
2191     my $section = '';
2192
2193     my @lines = split(/\n/, $raw);
2194     foreach my $line (@lines) {
2195         next if $line =~ m/^\s*$/;
2196
2197         if ($line =~ m/^\[PENDING\]\s*$/i) {
2198             $section = 'pending';
2199             if (defined($descr)) {
2200                 $descr =~ s/\s+$//;
2201                 $conf->{description} = $descr;
2202             }
2203             $descr = undef;
2204             $conf = $res->{$section} = {};
2205             next;
2206
2207         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2208             $section = $1;
2209             if (defined($descr)) {
2210                 $descr =~ s/\s+$//;
2211                 $conf->{description} = $descr;
2212             }
2213             $descr = undef;
2214             $conf = $res->{snapshots}->{$section} = {};
2215             next;
2216         }
2217
2218         if ($line =~ m/^\#(.*)\s*$/) {
2219             $descr = '' if !defined($descr);
2220             $descr .= PVE::Tools::decode_text($1) . "\n";
2221             next;
2222         }
2223
2224         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2225             $descr = '' if !defined($descr);
2226             $descr .= PVE::Tools::decode_text($2);
2227         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2228             $conf->{snapstate} = $1;
2229         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2230             my $key = $1;
2231             my $value = $2;
2232             $conf->{$key} = $value;
2233         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2234             my $value = $1;
2235             if ($section eq 'pending') {
2236                 $conf->{delete} = $value; # we parse this later
2237             } else {
2238                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2239             }
2240         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2241             my $key = $1;
2242             my $value = $2;
2243             eval { $value = check_type($key, $value); };
2244             if ($@) {
2245                 warn "vm $vmid - unable to parse value of '$key' - $@";
2246             } else {
2247                 $key = 'ide2' if $key eq 'cdrom';
2248                 my $fmt = $confdesc->{$key}->{format};
2249                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2250                     my $v = parse_drive($key, $value);
2251                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2252                         $v->{file} = $volid;
2253                         $value = print_drive($vmid, $v);
2254                     } else {
2255                         warn "vm $vmid - unable to parse value of '$key'\n";
2256                         next;
2257                     }
2258                 }
2259
2260                 $conf->{$key} = $value;
2261             }
2262         }
2263     }
2264
2265     if (defined($descr)) {
2266         $descr =~ s/\s+$//;
2267         $conf->{description} = $descr;
2268     }
2269     delete $res->{snapstate}; # just to be sure
2270
2271     return $res;
2272 }
2273
2274 sub write_vm_config {
2275     my ($filename, $conf) = @_;
2276
2277     delete $conf->{snapstate}; # just to be sure
2278
2279     if ($conf->{cdrom}) {
2280         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2281         $conf->{ide2} = $conf->{cdrom};
2282         delete $conf->{cdrom};
2283     }
2284
2285     # we do not use 'smp' any longer
2286     if ($conf->{sockets}) {
2287         delete $conf->{smp};
2288     } elsif ($conf->{smp}) {
2289         $conf->{sockets} = $conf->{smp};
2290         delete $conf->{cores};
2291         delete $conf->{smp};
2292     }
2293
2294     my $used_volids = {};
2295
2296     my $cleanup_config = sub {
2297         my ($cref, $pending, $snapname) = @_;
2298
2299         foreach my $key (keys %$cref) {
2300             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2301                 $key eq 'snapstate' || $key eq 'pending';
2302             my $value = $cref->{$key};
2303             if ($key eq 'delete') {
2304                 die "propertry 'delete' is only allowed in [PENDING]\n"
2305                     if !$pending;
2306                 # fixme: check syntax?
2307                 next;
2308             }
2309             eval { $value = check_type($key, $value); };
2310             die "unable to parse value of '$key' - $@" if $@;
2311
2312             $cref->{$key} = $value;
2313
2314             if (!$snapname && is_valid_drivename($key)) {
2315                 my $drive = parse_drive($key, $value);
2316                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2317             }
2318         }
2319     };
2320
2321     &$cleanup_config($conf);
2322
2323     &$cleanup_config($conf->{pending}, 1);
2324
2325     foreach my $snapname (keys %{$conf->{snapshots}}) {
2326         die "internal error" if $snapname eq 'pending';
2327         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2328     }
2329
2330     # remove 'unusedX' settings if we re-add a volume
2331     foreach my $key (keys %$conf) {
2332         my $value = $conf->{$key};
2333         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2334             delete $conf->{$key};
2335         }
2336     }
2337
2338     my $generate_raw_config = sub {
2339         my ($conf, $pending) = @_;
2340
2341         my $raw = '';
2342
2343         # add description as comment to top of file
2344         if (defined(my $descr = $conf->{description})) {
2345             if ($descr) {
2346                 foreach my $cl (split(/\n/, $descr)) {
2347                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2348                 }
2349             } else {
2350                 $raw .= "#\n" if $pending;
2351             }
2352         }
2353
2354         foreach my $key (sort keys %$conf) {
2355             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2356             $raw .= "$key: $conf->{$key}\n";
2357         }
2358         return $raw;
2359     };
2360
2361     my $raw = &$generate_raw_config($conf);
2362
2363     if (scalar(keys %{$conf->{pending}})){
2364         $raw .= "\n[PENDING]\n";
2365         $raw .= &$generate_raw_config($conf->{pending}, 1);
2366     }
2367
2368     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2369         $raw .= "\n[$snapname]\n";
2370         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2371     }
2372
2373     return $raw;
2374 }
2375
2376 sub load_defaults {
2377
2378     my $res = {};
2379
2380     # we use static defaults from our JSON schema configuration
2381     foreach my $key (keys %$confdesc) {
2382         if (defined(my $default = $confdesc->{$key}->{default})) {
2383             $res->{$key} = $default;
2384         }
2385     }
2386
2387     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2388     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2389
2390     return $res;
2391 }
2392
2393 sub config_list {
2394     my $vmlist = PVE::Cluster::get_vmlist();
2395     my $res = {};
2396     return $res if !$vmlist || !$vmlist->{ids};
2397     my $ids = $vmlist->{ids};
2398
2399     foreach my $vmid (keys %$ids) {
2400         my $d = $ids->{$vmid};
2401         next if !$d->{node} || $d->{node} ne $nodename;
2402         next if !$d->{type} || $d->{type} ne 'qemu';
2403         $res->{$vmid}->{exists} = 1;
2404     }
2405     return $res;
2406 }
2407
2408 # test if VM uses local resources (to prevent migration)
2409 sub check_local_resources {
2410     my ($conf, $noerr) = @_;
2411
2412     my $loc_res = 0;
2413
2414     $loc_res = 1 if $conf->{hostusb}; # old syntax
2415     $loc_res = 1 if $conf->{hostpci}; # old syntax
2416
2417     foreach my $k (keys %$conf) {
2418         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2419         # sockets are safe: they will recreated be on the target side post-migrate
2420         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2421         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2422     }
2423
2424     die "VM uses local resources\n" if $loc_res && !$noerr;
2425
2426     return $loc_res;
2427 }
2428
2429 # check if used storages are available on all nodes (use by migrate)
2430 sub check_storage_availability {
2431     my ($storecfg, $conf, $node) = @_;
2432
2433     foreach_drive($conf, sub {
2434         my ($ds, $drive) = @_;
2435
2436         my $volid = $drive->{file};
2437         return if !$volid;
2438
2439         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2440         return if !$sid;
2441
2442         # check if storage is available on both nodes
2443         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2444         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2445    });
2446 }
2447
2448 # list nodes where all VM images are available (used by has_feature API)
2449 sub shared_nodes {
2450     my ($conf, $storecfg) = @_;
2451
2452     my $nodelist = PVE::Cluster::get_nodelist();
2453     my $nodehash = { map { $_ => 1 } @$nodelist };
2454     my $nodename = PVE::INotify::nodename();
2455
2456     foreach_drive($conf, sub {
2457         my ($ds, $drive) = @_;
2458
2459         my $volid = $drive->{file};
2460         return if !$volid;
2461
2462         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2463         if ($storeid) {
2464             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2465             if ($scfg->{disable}) {
2466                 $nodehash = {};
2467             } elsif (my $avail = $scfg->{nodes}) {
2468                 foreach my $node (keys %$nodehash) {
2469                     delete $nodehash->{$node} if !$avail->{$node};
2470                 }
2471             } elsif (!$scfg->{shared}) {
2472                 foreach my $node (keys %$nodehash) {
2473                     delete $nodehash->{$node} if $node ne $nodename
2474                 }
2475             }
2476         }
2477     });
2478
2479     return $nodehash
2480 }
2481
2482 sub check_cmdline {
2483     my ($pidfile, $pid) = @_;
2484
2485     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2486     if (defined($fh)) {
2487         my $line = <$fh>;
2488         $fh->close;
2489         return undef if !$line;
2490         my @param = split(/\0/, $line);
2491
2492         my $cmd = $param[0];
2493         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2494
2495         for (my $i = 0; $i < scalar (@param); $i++) {
2496             my $p = $param[$i];
2497             next if !$p;
2498             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2499                 my $p = $param[$i+1];
2500                 return 1 if $p && ($p eq $pidfile);
2501                 return undef;
2502             }
2503         }
2504     }
2505     return undef;
2506 }
2507
2508 sub check_running {
2509     my ($vmid, $nocheck, $node) = @_;
2510
2511     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2512
2513     die "unable to find configuration file for VM $vmid - no such machine\n"
2514         if !$nocheck && ! -f $filename;
2515
2516     my $pidfile = pidfile_name($vmid);
2517
2518     if (my $fd = IO::File->new("<$pidfile")) {
2519         my $st = stat($fd);
2520         my $line = <$fd>;
2521         close($fd);
2522
2523         my $mtime = $st->mtime;
2524         if ($mtime > time()) {
2525             warn "file '$filename' modified in future\n";
2526         }
2527
2528         if ($line =~ m/^(\d+)$/) {
2529             my $pid = $1;
2530             if (check_cmdline($pidfile, $pid)) {
2531                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2532                     return $pid;
2533                 }
2534             }
2535         }
2536     }
2537
2538     return undef;
2539 }
2540
2541 sub vzlist {
2542
2543     my $vzlist = config_list();
2544
2545     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2546
2547     while (defined(my $de = $fd->read)) {
2548         next if $de !~ m/^(\d+)\.pid$/;
2549         my $vmid = $1;
2550         next if !defined($vzlist->{$vmid});
2551         if (my $pid = check_running($vmid)) {
2552             $vzlist->{$vmid}->{pid} = $pid;
2553         }
2554     }
2555
2556     return $vzlist;
2557 }
2558
2559 sub disksize {
2560     my ($storecfg, $conf) = @_;
2561
2562     my $bootdisk = $conf->{bootdisk};
2563     return undef if !$bootdisk;
2564     return undef if !is_valid_drivename($bootdisk);
2565
2566     return undef if !$conf->{$bootdisk};
2567
2568     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2569     return undef if !defined($drive);
2570
2571     return undef if drive_is_cdrom($drive);
2572
2573     my $volid = $drive->{file};
2574     return undef if !$volid;
2575
2576     return $drive->{size};
2577 }
2578
2579 my $last_proc_pid_stat;
2580
2581 # get VM status information
2582 # This must be fast and should not block ($full == false)
2583 # We only query KVM using QMP if $full == true (this can be slow)
2584 sub vmstatus {
2585     my ($opt_vmid, $full) = @_;
2586
2587     my $res = {};
2588
2589     my $storecfg = PVE::Storage::config();
2590
2591     my $list = vzlist();
2592     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2593
2594     my $cpucount = $cpuinfo->{cpus} || 1;
2595
2596     foreach my $vmid (keys %$list) {
2597         next if $opt_vmid && ($vmid ne $opt_vmid);
2598
2599         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2600         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2601
2602         my $d = {};
2603         $d->{pid} = $list->{$vmid}->{pid};
2604
2605         # fixme: better status?
2606         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2607
2608         my $size = disksize($storecfg, $conf);
2609         if (defined($size)) {
2610             $d->{disk} = 0; # no info available
2611             $d->{maxdisk} = $size;
2612         } else {
2613             $d->{disk} = 0;
2614             $d->{maxdisk} = 0;
2615         }
2616
2617         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2618         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2619         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2620
2621         $d->{name} = $conf->{name} || "VM $vmid";
2622         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2623
2624         if ($conf->{balloon}) {
2625             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2626             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2627         }
2628
2629         $d->{uptime} = 0;
2630         $d->{cpu} = 0;
2631         $d->{mem} = 0;
2632
2633         $d->{netout} = 0;
2634         $d->{netin} = 0;
2635
2636         $d->{diskread} = 0;
2637         $d->{diskwrite} = 0;
2638
2639         $d->{template} = PVE::QemuConfig->is_template($conf);
2640
2641         $res->{$vmid} = $d;
2642     }
2643
2644     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2645     foreach my $dev (keys %$netdev) {
2646         next if $dev !~ m/^tap([1-9]\d*)i/;
2647         my $vmid = $1;
2648         my $d = $res->{$vmid};
2649         next if !$d;
2650
2651         $d->{netout} += $netdev->{$dev}->{receive};
2652         $d->{netin} += $netdev->{$dev}->{transmit};
2653
2654         if ($full) {
2655             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2656             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2657         }
2658
2659     }
2660
2661     my $ctime = gettimeofday;
2662
2663     foreach my $vmid (keys %$list) {
2664
2665         my $d = $res->{$vmid};
2666         my $pid = $d->{pid};
2667         next if !$pid;
2668
2669         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2670         next if !$pstat; # not running
2671
2672         my $used = $pstat->{utime} + $pstat->{stime};
2673
2674         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2675
2676         if ($pstat->{vsize}) {
2677             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2678         }
2679
2680         my $old = $last_proc_pid_stat->{$pid};
2681         if (!$old) {
2682             $last_proc_pid_stat->{$pid} = {
2683                 time => $ctime,
2684                 used => $used,
2685                 cpu => 0,
2686             };
2687             next;
2688         }
2689
2690         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2691
2692         if ($dtime > 1000) {
2693             my $dutime = $used -  $old->{used};
2694
2695             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2696             $last_proc_pid_stat->{$pid} = {
2697                 time => $ctime,
2698                 used => $used,
2699                 cpu => $d->{cpu},
2700             };
2701         } else {
2702             $d->{cpu} = $old->{cpu};
2703         }
2704     }
2705
2706     return $res if !$full;
2707
2708     my $qmpclient = PVE::QMPClient->new();
2709
2710     my $ballooncb = sub {
2711         my ($vmid, $resp) = @_;
2712
2713         my $info = $resp->{'return'};
2714         return if !$info->{max_mem};
2715
2716         my $d = $res->{$vmid};
2717
2718         # use memory assigned to VM
2719         $d->{maxmem} = $info->{max_mem};
2720         $d->{balloon} = $info->{actual};
2721
2722         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2723             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2724             $d->{freemem} = $info->{free_mem};
2725         }
2726
2727         $d->{ballooninfo} = $info;
2728     };
2729
2730     my $blockstatscb = sub {
2731         my ($vmid, $resp) = @_;
2732         my $data = $resp->{'return'} || [];
2733         my $totalrdbytes = 0;
2734         my $totalwrbytes = 0;
2735
2736         for my $blockstat (@$data) {
2737             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2738             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2739
2740             $blockstat->{device} =~ s/drive-//;
2741             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2742         }
2743         $res->{$vmid}->{diskread} = $totalrdbytes;
2744         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2745     };
2746
2747     my $statuscb = sub {
2748         my ($vmid, $resp) = @_;
2749
2750         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2751         # this fails if ballon driver is not loaded, so this must be
2752         # the last commnand (following command are aborted if this fails).
2753         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2754
2755         my $status = 'unknown';
2756         if (!defined($status = $resp->{'return'}->{status})) {
2757             warn "unable to get VM status\n";
2758             return;
2759         }
2760
2761         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2762     };
2763
2764     foreach my $vmid (keys %$list) {
2765         next if $opt_vmid && ($vmid ne $opt_vmid);
2766         next if !$res->{$vmid}->{pid}; # not running
2767         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2768     }
2769
2770     $qmpclient->queue_execute(undef, 2);
2771
2772     foreach my $vmid (keys %$list) {
2773         next if $opt_vmid && ($vmid ne $opt_vmid);
2774         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2775     }
2776
2777     return $res;
2778 }
2779
2780 sub foreach_drive {
2781     my ($conf, $func, @param) = @_;
2782
2783     foreach my $ds (valid_drive_names()) {
2784         next if !defined($conf->{$ds});
2785
2786         my $drive = parse_drive($ds, $conf->{$ds});
2787         next if !$drive;
2788
2789         &$func($ds, $drive, @param);
2790     }
2791 }
2792
2793 sub foreach_volid {
2794     my ($conf, $func, @param) = @_;
2795
2796     my $volhash = {};
2797
2798     my $test_volid = sub {
2799         my ($volid, $is_cdrom, $replicate, $snapname) = @_;
2800
2801         return if !$volid;
2802
2803         $volhash->{$volid}->{cdrom} //= 1;
2804         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2805
2806         $volhash->{$volid}->{replicate} //= 0;
2807         $volhash->{$volid}->{replicate} = 1 if $replicate;
2808
2809         $volhash->{$volid}->{referenced_in_config} //= 0;
2810         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
2811
2812         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
2813             if defined($snapname);
2814     };
2815
2816     foreach_drive($conf, sub {
2817         my ($ds, $drive) = @_;
2818         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, undef);
2819     });
2820
2821     foreach my $snapname (keys %{$conf->{snapshots}}) {
2822         my $snap = $conf->{snapshots}->{$snapname};
2823         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
2824         foreach_drive($snap, sub {
2825             my ($ds, $drive) = @_;
2826             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $snapname);
2827         });
2828     }
2829
2830     foreach my $volid (keys %$volhash) {
2831         &$func($volid, $volhash->{$volid}, @param);
2832     }
2833 }
2834
2835 sub vga_conf_has_spice {
2836     my ($vga) = @_;
2837
2838     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2839
2840     return $1 || 1;
2841 }
2842
2843 sub config_to_command {
2844     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2845
2846     my $cmd = [];
2847     my $globalFlags = [];
2848     my $machineFlags = [];
2849     my $rtcFlags = [];
2850     my $cpuFlags = [];
2851     my $devices = [];
2852     my $pciaddr = '';
2853     my $bridges = {};
2854     my $kvmver = kvm_user_version();
2855     my $vernum = 0; # unknown
2856     my $ostype = $conf->{ostype};
2857     my $winversion = windows_version($ostype);
2858     my $kvm = $conf->{kvm} // 1;
2859
2860     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
2861
2862     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2863         $vernum = $1*1000000+$2*1000;
2864     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2865         $vernum = $1*1000000+$2*1000+$3;
2866     }
2867
2868     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2869
2870     my $have_ovz = -f '/proc/vz/vestat';
2871
2872     my $q35 = machine_type_is_q35($conf);
2873     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2874     my $machine_type = $forcemachine || $conf->{machine};
2875     my $use_old_bios_files = undef;
2876     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2877
2878     my $cpuunits = defined($conf->{cpuunits}) ?
2879             $conf->{cpuunits} : $defaults->{cpuunits};
2880
2881     push @$cmd, '/usr/bin/kvm';
2882
2883     push @$cmd, '-id', $vmid;
2884
2885     my $use_virtio = 0;
2886
2887     my $qmpsocket = qmp_socket($vmid);
2888     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2889     push @$cmd, '-mon', "chardev=qmp,mode=control";
2890
2891
2892     push @$cmd, '-pidfile' , pidfile_name($vmid);
2893
2894     push @$cmd, '-daemonize';
2895
2896     if ($conf->{smbios1}) {
2897         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2898     }
2899
2900     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2901         die "uefi base image not found\n" if ! -f $OVMF_CODE;
2902
2903         my $path;
2904         my $format = 'raw';
2905         if (my $efidisk = $conf->{efidisk0}) {
2906             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
2907             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2908             if ($storeid) {
2909                 $path = PVE::Storage::path($storecfg, $d->{file});
2910             } else {
2911                 $path = $d->{file};
2912             }
2913             $format = $d->{format} if $d->{format};
2914         } else {
2915             warn "no efidisk configured! Using temporary efivars disk.\n";
2916             $path = "/tmp/$vmid-ovmf.fd";
2917             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
2918         }
2919
2920         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
2921         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
2922     }
2923
2924
2925     # add usb controllers
2926     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2927     push @$devices, @usbcontrollers if @usbcontrollers;
2928     my $vga = $conf->{vga};
2929
2930     my $qxlnum = vga_conf_has_spice($vga);
2931     $vga = 'qxl' if $qxlnum;
2932
2933     if (!$vga) {
2934         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
2935             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
2936         } else {
2937             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
2938         }
2939     }
2940
2941     # enable absolute mouse coordinates (needed by vnc)
2942     my $tablet;
2943     if (defined($conf->{tablet})) {
2944         $tablet = $conf->{tablet};
2945     } else {
2946         $tablet = $defaults->{tablet};
2947         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2948         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2949     }
2950
2951     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2952
2953     my $kvm_off = 0;
2954     my $gpu_passthrough;
2955
2956     # host pci devices
2957     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2958         my $d = parse_hostpci($conf->{"hostpci$i"});
2959         next if !$d;
2960
2961         my $pcie = $d->{pcie};
2962         if($pcie){
2963             die "q35 machine model is not enabled" if !$q35;
2964             $pciaddr = print_pcie_addr("hostpci$i");
2965         }else{
2966             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2967         }
2968
2969         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2970         my $romfile = $d->{romfile};
2971
2972         my $xvga = '';
2973         if ($d->{'x-vga'}) {
2974             $xvga = ',x-vga=on';
2975             $kvm_off = 1;
2976             $vga = 'none';
2977             $gpu_passthrough = 1;
2978
2979             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2980                 $xvga = "";
2981             }
2982         }
2983         my $pcidevices = $d->{pciid};
2984         my $multifunction = 1 if @$pcidevices > 1;
2985
2986         my $j=0;
2987         foreach my $pcidevice (@$pcidevices) {
2988
2989             my $id = "hostpci$i";
2990             $id .= ".$j" if $multifunction;
2991             my $addr = $pciaddr;
2992             $addr .= ".$j" if $multifunction;
2993             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2994
2995             if($j == 0){
2996                 $devicestr .= "$rombar$xvga";
2997                 $devicestr .= ",multifunction=on" if $multifunction;
2998                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
2999             }
3000
3001             push @$devices, '-device', $devicestr;
3002             $j++;
3003         }
3004     }
3005
3006     # usb devices
3007     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3008     push @$devices, @usbdevices if @usbdevices;
3009     # serial devices
3010     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3011         if (my $path = $conf->{"serial$i"}) {
3012             if ($path eq 'socket') {
3013                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3014                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3015                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3016             } else {
3017                 die "no such serial device\n" if ! -c $path;
3018                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3019                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3020             }
3021         }
3022     }
3023
3024     # parallel devices
3025     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3026         if (my $path = $conf->{"parallel$i"}) {
3027             die "no such parallel device\n" if ! -c $path;
3028             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3029             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3030             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3031         }
3032     }
3033
3034     my $vmname = $conf->{name} || "vm$vmid";
3035
3036     push @$cmd, '-name', $vmname;
3037
3038     my $sockets = 1;
3039     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3040     $sockets = $conf->{sockets} if  $conf->{sockets};
3041
3042     my $cores = $conf->{cores} || 1;
3043
3044     my $maxcpus = $sockets * $cores;
3045
3046     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3047
3048     my $allowed_vcpus = $cpuinfo->{cpus};
3049
3050     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3051         if ($allowed_vcpus < $maxcpus);
3052
3053     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3054
3055         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3056         for (my $i = 2; $i <= $vcpus; $i++)  {
3057             my $cpustr = print_cpu_device($conf,$i);
3058             push @$cmd, '-device', $cpustr;
3059         }
3060
3061     } else {
3062
3063         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3064     }
3065     push @$cmd, '-nodefaults';
3066
3067     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3068
3069     my $bootindex_hash = {};
3070     my $i = 1;
3071     foreach my $o (split(//, $bootorder)) {
3072         $bootindex_hash->{$o} = $i*100;
3073         $i++;
3074     }
3075
3076     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3077
3078     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3079
3080     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3081
3082     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3083
3084     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3085         my $socket = vnc_socket($vmid);
3086         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3087     } else {
3088         push @$cmd, '-nographic';
3089     }
3090
3091     # time drift fix
3092     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3093
3094     my $useLocaltime = $conf->{localtime};
3095
3096     if ($winversion >= 5) { # windows
3097         $useLocaltime = 1 if !defined($conf->{localtime});
3098
3099         # use time drift fix when acpi is enabled
3100         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3101             $tdf = 1 if !defined($conf->{tdf});
3102         }
3103     }
3104
3105     if ($winversion >= 6) {
3106         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3107         push @$cmd, '-no-hpet';
3108     }
3109
3110     push @$rtcFlags, 'driftfix=slew' if $tdf;
3111
3112     if (!$kvm) {
3113         push @$machineFlags, 'accel=tcg';
3114     }
3115
3116     if ($machine_type) {
3117         push @$machineFlags, "type=${machine_type}";
3118     }
3119
3120     if ($conf->{startdate}) {
3121         push @$rtcFlags, "base=$conf->{startdate}";
3122     } elsif ($useLocaltime) {
3123         push @$rtcFlags, 'base=localtime';
3124     }
3125
3126     my $cpu = $kvm ? "kvm64" : "qemu64";
3127     if (my $cputype = $conf->{cpu}) {
3128         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3129             or die "Cannot parse cpu description: $cputype\n";
3130         $cpu = $cpuconf->{cputype};
3131         $kvm_off = 1 if $cpuconf->{hidden};
3132     }
3133
3134     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3135
3136     push @$cpuFlags , '-x2apic'
3137         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3138
3139     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3140
3141     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3142
3143     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3144
3145         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3146         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3147     }
3148
3149     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3150
3151     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3152
3153     push @$cpuFlags, 'kvm=off' if $kvm_off;
3154
3155     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3156         die "internal error"; # should not happen
3157
3158     push @$cpuFlags, "vendor=${cpu_vendor}"
3159         if $cpu_vendor ne 'default';
3160
3161     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3162
3163     push @$cmd, '-cpu', $cpu;
3164
3165     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3166
3167     push @$cmd, '-S' if $conf->{freeze};
3168
3169     # set keyboard layout
3170     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3171     push @$cmd, '-k', $kb if $kb;
3172
3173     # enable sound
3174     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3175     #push @$cmd, '-soundhw', 'es1370';
3176     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3177
3178     if($conf->{agent}) {
3179         my $qgasocket = qmp_socket($vmid, 1);
3180         my $pciaddr = print_pci_addr("qga0", $bridges);
3181         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3182         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3183         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3184     }
3185
3186     my $spice_port;
3187
3188     if ($qxlnum) {
3189         if ($qxlnum > 1) {
3190             if ($winversion){
3191                 for(my $i = 1; $i < $qxlnum; $i++){
3192                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3193                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3194                 }
3195             } else {
3196                 # assume other OS works like Linux
3197                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3198                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3199             }
3200         }
3201
3202         my $pciaddr = print_pci_addr("spice", $bridges);
3203
3204         my $nodename = PVE::INotify::nodename();
3205         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3206         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3207         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3208         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3209         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3210
3211         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3212
3213         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3214         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3215         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3216     }
3217
3218     # enable balloon by default, unless explicitly disabled
3219     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3220         $pciaddr = print_pci_addr("balloon0", $bridges);
3221         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3222     }
3223
3224     if ($conf->{watchdog}) {
3225         my $wdopts = parse_watchdog($conf->{watchdog});
3226         $pciaddr = print_pci_addr("watchdog", $bridges);
3227         my $watchdog = $wdopts->{model} || 'i6300esb';
3228         push @$devices, '-device', "$watchdog$pciaddr";
3229         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3230     }
3231
3232     my $vollist = [];
3233     my $scsicontroller = {};
3234     my $ahcicontroller = {};
3235     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3236
3237     # Add iscsi initiator name if available
3238     if (my $initiator = get_initiator_name()) {
3239         push @$devices, '-iscsi', "initiator-name=$initiator";
3240     }
3241
3242     foreach_drive($conf, sub {
3243         my ($ds, $drive) = @_;
3244
3245         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3246             push @$vollist, $drive->{file};
3247         }
3248
3249         # ignore efidisk here, already added in bios/fw handling code above
3250         return if $drive->{interface} eq 'efidisk';
3251
3252         $use_virtio = 1 if $ds =~ m/^virtio/;
3253
3254         if (drive_is_cdrom ($drive)) {
3255             if ($bootindex_hash->{d}) {
3256                 $drive->{bootindex} = $bootindex_hash->{d};
3257                 $bootindex_hash->{d} += 1;
3258             }
3259         } else {
3260             if ($bootindex_hash->{c}) {
3261                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3262                 $bootindex_hash->{c} += 1;
3263             }
3264         }
3265
3266         if($drive->{interface} eq 'virtio'){
3267            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3268         }
3269
3270         if ($drive->{interface} eq 'scsi') {
3271
3272             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3273
3274             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3275             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3276
3277             my $iothread = '';
3278             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3279                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3280                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3281             } elsif ($drive->{iothread}) {
3282                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3283             }
3284
3285             my $queues = '';
3286             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3287                 $queues = ",num_queues=$drive->{queues}";
3288             }
3289
3290             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3291             $scsicontroller->{$controller}=1;
3292         }
3293
3294         if ($drive->{interface} eq 'sata') {
3295            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3296            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3297            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3298            $ahcicontroller->{$controller}=1;
3299         }
3300
3301         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3302         push @$devices, '-drive',$drive_cmd;
3303         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3304     });
3305
3306     for (my $i = 0; $i < $MAX_NETS; $i++) {
3307          next if !$conf->{"net$i"};
3308          my $d = parse_net($conf->{"net$i"});
3309          next if !$d;
3310
3311          $use_virtio = 1 if $d->{model} eq 'virtio';
3312
3313          if ($bootindex_hash->{n}) {
3314             $d->{bootindex} = $bootindex_hash->{n};
3315             $bootindex_hash->{n} += 1;
3316          }
3317
3318          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3319          push @$devices, '-netdev', $netdevfull;
3320
3321          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3322          push @$devices, '-device', $netdevicefull;
3323     }
3324
3325     if (!$q35) {
3326         # add pci bridges
3327         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3328            $bridges->{1} = 1;
3329            $bridges->{2} = 1;
3330         }
3331
3332         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3333
3334         while (my ($k, $v) = each %$bridges) {
3335             $pciaddr = print_pci_addr("pci.$k");
3336             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3337         }
3338     }
3339
3340     # add custom args
3341     if ($conf->{args}) {
3342         my $aa = PVE::Tools::split_args($conf->{args});
3343         push @$cmd, @$aa;
3344     }
3345
3346     push @$cmd, @$devices;
3347     push @$cmd, '-rtc', join(',', @$rtcFlags)
3348         if scalar(@$rtcFlags);
3349     push @$cmd, '-machine', join(',', @$machineFlags)
3350         if scalar(@$machineFlags);
3351     push @$cmd, '-global', join(',', @$globalFlags)
3352         if scalar(@$globalFlags);
3353
3354     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3355 }
3356
3357 sub vnc_socket {
3358     my ($vmid) = @_;
3359     return "${var_run_tmpdir}/$vmid.vnc";
3360 }
3361
3362 sub spice_port {
3363     my ($vmid) = @_;
3364
3365     my $res = vm_mon_cmd($vmid, 'query-spice');
3366
3367     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3368 }
3369
3370 sub qmp_socket {
3371     my ($vmid, $qga) = @_;
3372     my $sockettype = $qga ? 'qga' : 'qmp';
3373     return "${var_run_tmpdir}/$vmid.$sockettype";
3374 }
3375
3376 sub pidfile_name {
3377     my ($vmid) = @_;
3378     return "${var_run_tmpdir}/$vmid.pid";
3379 }
3380
3381 sub vm_devices_list {
3382     my ($vmid) = @_;
3383
3384     my $res = vm_mon_cmd($vmid, 'query-pci');
3385     my $devices = {};
3386     foreach my $pcibus (@$res) {
3387         foreach my $device (@{$pcibus->{devices}}) {
3388             next if !$device->{'qdev_id'};
3389             if ($device->{'pci_bridge'}) {
3390                 $devices->{$device->{'qdev_id'}} = 1;
3391                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3392                     next if !$bridge_device->{'qdev_id'};
3393                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3394                     $devices->{$device->{'qdev_id'}}++;
3395                 }
3396             } else {
3397                 $devices->{$device->{'qdev_id'}} = 1;
3398             }
3399         }
3400     }
3401
3402     my $resblock = vm_mon_cmd($vmid, 'query-block');
3403     foreach my $block (@$resblock) {
3404         if($block->{device} =~ m/^drive-(\S+)/){
3405                 $devices->{$1} = 1;
3406         }
3407     }
3408
3409     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3410     foreach my $mice (@$resmice) {
3411         if ($mice->{name} eq 'QEMU HID Tablet') {
3412             $devices->{tablet} = 1;
3413             last;
3414         }
3415     }
3416
3417     # for usb devices there is no query-usb
3418     # but we can iterate over the entries in
3419     # qom-list path=/machine/peripheral
3420     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3421     foreach my $per (@$resperipheral) {
3422         if ($per->{name} =~ m/^usb\d+$/) {
3423             $devices->{$per->{name}} = 1;
3424         }
3425     }
3426
3427     return $devices;
3428 }
3429
3430 sub vm_deviceplug {
3431     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3432
3433     my $q35 = machine_type_is_q35($conf);
3434
3435     my $devices_list = vm_devices_list($vmid);
3436     return 1 if defined($devices_list->{$deviceid});
3437
3438     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3439
3440     if ($deviceid eq 'tablet') {
3441
3442         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3443
3444     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3445
3446         die "usb hotplug currently not reliable\n";
3447         # since we can't reliably hot unplug all added usb devices
3448         # and usb passthrough disables live migration
3449         # we disable usb hotplugging for now
3450         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3451
3452     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3453
3454         qemu_iothread_add($vmid, $deviceid, $device);
3455
3456         qemu_driveadd($storecfg, $vmid, $device);
3457         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3458
3459         qemu_deviceadd($vmid, $devicefull);
3460         eval { qemu_deviceaddverify($vmid, $deviceid); };
3461         if (my $err = $@) {
3462             eval { qemu_drivedel($vmid, $deviceid); };
3463             warn $@ if $@;
3464             die $err;
3465         }
3466
3467     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3468
3469
3470         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3471         my $pciaddr = print_pci_addr($deviceid);
3472         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3473
3474         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3475
3476         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3477             qemu_iothread_add($vmid, $deviceid, $device);
3478             $devicefull .= ",iothread=iothread-$deviceid";
3479         }
3480
3481         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3482             $devicefull .= ",num_queues=$device->{queues}";
3483         }
3484
3485         qemu_deviceadd($vmid, $devicefull);
3486         qemu_deviceaddverify($vmid, $deviceid);
3487
3488     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3489
3490         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3491         qemu_driveadd($storecfg, $vmid, $device);
3492
3493         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3494         eval { qemu_deviceadd($vmid, $devicefull); };
3495         if (my $err = $@) {
3496             eval { qemu_drivedel($vmid, $deviceid); };
3497             warn $@ if $@;
3498             die $err;
3499         }
3500
3501     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3502
3503         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3504
3505         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3506         my $use_old_bios_files = undef;
3507         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3508
3509         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3510         qemu_deviceadd($vmid, $netdevicefull);
3511         eval { qemu_deviceaddverify($vmid, $deviceid); };
3512         if (my $err = $@) {
3513             eval { qemu_netdevdel($vmid, $deviceid); };
3514             warn $@ if $@;
3515             die $err;
3516         }
3517
3518     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3519
3520         my $bridgeid = $2;
3521         my $pciaddr = print_pci_addr($deviceid);
3522         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3523
3524         qemu_deviceadd($vmid, $devicefull);
3525         qemu_deviceaddverify($vmid, $deviceid);
3526
3527     } else {
3528         die "can't hotplug device '$deviceid'\n";
3529     }
3530
3531     return 1;
3532 }
3533
3534 # fixme: this should raise exceptions on error!
3535 sub vm_deviceunplug {
3536     my ($vmid, $conf, $deviceid) = @_;
3537
3538     my $devices_list = vm_devices_list($vmid);
3539     return 1 if !defined($devices_list->{$deviceid});
3540
3541     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3542
3543     if ($deviceid eq 'tablet') {
3544
3545         qemu_devicedel($vmid, $deviceid);
3546
3547     } elsif ($deviceid =~ m/^usb\d+$/) {
3548
3549         die "usb hotplug currently not reliable\n";
3550         # when unplugging usb devices this way,
3551         # there may be remaining usb controllers/hubs
3552         # so we disable it for now
3553         qemu_devicedel($vmid, $deviceid);
3554         qemu_devicedelverify($vmid, $deviceid);
3555
3556     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3557
3558         qemu_devicedel($vmid, $deviceid);
3559         qemu_devicedelverify($vmid, $deviceid);
3560         qemu_drivedel($vmid, $deviceid);
3561         qemu_iothread_del($conf, $vmid, $deviceid);
3562
3563     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3564
3565         qemu_devicedel($vmid, $deviceid);
3566         qemu_devicedelverify($vmid, $deviceid);
3567         qemu_iothread_del($conf, $vmid, $deviceid);
3568
3569     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3570
3571         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3572         my $device = parse_drive($deviceid, $conf->{$deviceid});
3573         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3574
3575         qemu_devicedel($vmid, $deviceid);
3576         qemu_drivedel($vmid, $deviceid);
3577         qemu_deletescsihw($conf, $vmid, $deviceid);
3578
3579     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3580
3581         qemu_devicedel($vmid, $deviceid);
3582         qemu_devicedelverify($vmid, $deviceid);
3583         qemu_netdevdel($vmid, $deviceid);
3584
3585     } else {
3586         die "can't unplug device '$deviceid'\n";
3587     }
3588
3589     return 1;
3590 }
3591
3592 sub qemu_deviceadd {
3593     my ($vmid, $devicefull) = @_;
3594
3595     $devicefull = "driver=".$devicefull;
3596     my %options =  split(/[=,]/, $devicefull);
3597
3598     vm_mon_cmd($vmid, "device_add" , %options);
3599 }
3600
3601 sub qemu_devicedel {
3602     my ($vmid, $deviceid) = @_;
3603
3604     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3605 }
3606
3607 sub qemu_iothread_add {
3608     my($vmid, $deviceid, $device) = @_;
3609
3610     if ($device->{iothread}) {
3611         my $iothreads = vm_iothreads_list($vmid);
3612         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3613     }
3614 }
3615
3616 sub qemu_iothread_del {
3617     my($conf, $vmid, $deviceid) = @_;
3618
3619     my $device = parse_drive($deviceid, $conf->{$deviceid});
3620     if ($device->{iothread}) {
3621         my $iothreads = vm_iothreads_list($vmid);
3622         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3623     }
3624 }
3625
3626 sub qemu_objectadd {
3627     my($vmid, $objectid, $qomtype) = @_;
3628
3629     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3630
3631     return 1;
3632 }
3633
3634 sub qemu_objectdel {
3635     my($vmid, $objectid) = @_;
3636
3637     vm_mon_cmd($vmid, "object-del", id => $objectid);
3638
3639     return 1;
3640 }
3641
3642 sub qemu_driveadd {
3643     my ($storecfg, $vmid, $device) = @_;
3644
3645     my $drive = print_drive_full($storecfg, $vmid, $device);
3646     $drive =~ s/\\/\\\\/g;
3647     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3648
3649     # If the command succeeds qemu prints: "OK"
3650     return 1 if $ret =~ m/OK/s;
3651
3652     die "adding drive failed: $ret\n";
3653 }
3654
3655 sub qemu_drivedel {
3656     my($vmid, $deviceid) = @_;
3657
3658     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3659     $ret =~ s/^\s+//;
3660
3661     return 1 if $ret eq "";
3662
3663     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3664     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3665
3666     die "deleting drive $deviceid failed : $ret\n";
3667 }
3668
3669 sub qemu_deviceaddverify {
3670     my ($vmid, $deviceid) = @_;
3671
3672     for (my $i = 0; $i <= 5; $i++) {
3673          my $devices_list = vm_devices_list($vmid);
3674          return 1 if defined($devices_list->{$deviceid});
3675          sleep 1;
3676     }
3677
3678     die "error on hotplug device '$deviceid'\n";
3679 }
3680
3681
3682 sub qemu_devicedelverify {
3683     my ($vmid, $deviceid) = @_;
3684
3685     # need to verify that the device is correctly removed as device_del
3686     # is async and empty return is not reliable
3687
3688     for (my $i = 0; $i <= 5; $i++) {
3689          my $devices_list = vm_devices_list($vmid);
3690          return 1 if !defined($devices_list->{$deviceid});
3691          sleep 1;
3692     }
3693
3694     die "error on hot-unplugging device '$deviceid'\n";
3695 }
3696
3697 sub qemu_findorcreatescsihw {
3698     my ($storecfg, $conf, $vmid, $device) = @_;
3699
3700     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3701
3702     my $scsihwid="$controller_prefix$controller";
3703     my $devices_list = vm_devices_list($vmid);
3704
3705     if(!defined($devices_list->{$scsihwid})) {
3706         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3707     }
3708
3709     return 1;
3710 }
3711
3712 sub qemu_deletescsihw {
3713     my ($conf, $vmid, $opt) = @_;
3714
3715     my $device = parse_drive($opt, $conf->{$opt});
3716
3717     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3718         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3719         return 1;
3720     }
3721
3722     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3723
3724     my $devices_list = vm_devices_list($vmid);
3725     foreach my $opt (keys %{$devices_list}) {
3726         if (PVE::QemuServer::is_valid_drivename($opt)) {
3727             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3728             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3729                 return 1;
3730             }
3731         }
3732     }
3733
3734     my $scsihwid="scsihw$controller";
3735
3736     vm_deviceunplug($vmid, $conf, $scsihwid);
3737
3738     return 1;
3739 }
3740
3741 sub qemu_add_pci_bridge {
3742     my ($storecfg, $conf, $vmid, $device) = @_;
3743
3744     my $bridges = {};
3745
3746     my $bridgeid;
3747
3748     print_pci_addr($device, $bridges);
3749
3750     while (my ($k, $v) = each %$bridges) {
3751         $bridgeid = $k;
3752     }
3753     return 1 if !defined($bridgeid) || $bridgeid < 1;
3754
3755     my $bridge = "pci.$bridgeid";
3756     my $devices_list = vm_devices_list($vmid);
3757
3758     if (!defined($devices_list->{$bridge})) {
3759         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3760     }
3761
3762     return 1;
3763 }
3764
3765 sub qemu_set_link_status {
3766     my ($vmid, $device, $up) = @_;
3767
3768     vm_mon_cmd($vmid, "set_link", name => $device,
3769                up => $up ? JSON::true : JSON::false);
3770 }
3771
3772 sub qemu_netdevadd {
3773     my ($vmid, $conf, $device, $deviceid) = @_;
3774
3775     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3776     my %options =  split(/[=,]/, $netdev);
3777
3778     vm_mon_cmd($vmid, "netdev_add",  %options);
3779     return 1;
3780 }
3781
3782 sub qemu_netdevdel {
3783     my ($vmid, $deviceid) = @_;
3784
3785     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3786 }
3787
3788 sub qemu_usb_hotplug {
3789     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3790
3791     return if !$device;
3792
3793     # remove the old one first
3794     vm_deviceunplug($vmid, $conf, $deviceid);
3795
3796     # check if xhci controller is necessary and available
3797     if ($device->{usb3}) {
3798
3799         my $devicelist = vm_devices_list($vmid);
3800
3801         if (!$devicelist->{xhci}) {
3802             my $pciaddr = print_pci_addr("xhci");
3803             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3804         }
3805     }
3806     my $d = parse_usb_device($device->{host});
3807     $d->{usb3} = $device->{usb3};
3808
3809     # add the new one
3810     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3811 }
3812
3813 sub qemu_cpu_hotplug {
3814     my ($vmid, $conf, $vcpus) = @_;
3815
3816     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3817
3818     my $sockets = 1;
3819     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3820     $sockets = $conf->{sockets} if  $conf->{sockets};
3821     my $cores = $conf->{cores} || 1;
3822     my $maxcpus = $sockets * $cores;
3823
3824     $vcpus = $maxcpus if !$vcpus;
3825
3826     die "you can't add more vcpus than maxcpus\n"
3827         if $vcpus > $maxcpus;
3828
3829     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3830
3831     if ($vcpus < $currentvcpus) {
3832
3833         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3834
3835             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3836                 qemu_devicedel($vmid, "cpu$i");
3837                 my $retry = 0;
3838                 my $currentrunningvcpus = undef;
3839                 while (1) {
3840                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3841                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3842                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3843                     $retry++;
3844                     sleep 1;
3845                 }
3846                 #update conf after each succesfull cpu unplug
3847                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3848                 PVE::QemuConfig->write_config($vmid, $conf);
3849             }
3850         } else {
3851             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3852         }
3853
3854         return;
3855     }
3856
3857     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3858     die "vcpus in running vm does not match its configuration\n"
3859         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3860
3861     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3862
3863         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3864             my $cpustr = print_cpu_device($conf, $i);
3865             qemu_deviceadd($vmid, $cpustr);
3866
3867             my $retry = 0;
3868             my $currentrunningvcpus = undef;
3869             while (1) {
3870                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3871                 last if scalar(@{$currentrunningvcpus}) == $i;
3872                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3873                 sleep 1;
3874                 $retry++;
3875             }
3876             #update conf after each succesfull cpu hotplug
3877             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3878             PVE::QemuConfig->write_config($vmid, $conf);
3879         }
3880     } else {
3881
3882         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3883             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3884         }
3885     }
3886 }
3887
3888 sub qemu_block_set_io_throttle {
3889     my ($vmid, $deviceid,
3890         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3891         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3892         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3893         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3894
3895     return if !check_running($vmid) ;
3896
3897     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3898         bps => int($bps),
3899         bps_rd => int($bps_rd),
3900         bps_wr => int($bps_wr),
3901         iops => int($iops),
3902         iops_rd => int($iops_rd),
3903         iops_wr => int($iops_wr),
3904         bps_max => int($bps_max),
3905         bps_rd_max => int($bps_rd_max),
3906         bps_wr_max => int($bps_wr_max),
3907         iops_max => int($iops_max),
3908         iops_rd_max => int($iops_rd_max),
3909         iops_wr_max => int($iops_wr_max),
3910         bps_max_length => int($bps_max_length),
3911         bps_rd_max_length => int($bps_rd_max_length),
3912         bps_wr_max_length => int($bps_wr_max_length),
3913         iops_max_length => int($iops_max_length),
3914         iops_rd_max_length => int($iops_rd_max_length),
3915         iops_wr_max_length => int($iops_wr_max_length),
3916     );
3917
3918 }
3919
3920 # old code, only used to shutdown old VM after update
3921 sub __read_avail {
3922     my ($fh, $timeout) = @_;
3923
3924     my $sel = new IO::Select;
3925     $sel->add($fh);
3926
3927     my $res = '';
3928     my $buf;
3929
3930     my @ready;
3931     while (scalar (@ready = $sel->can_read($timeout))) {
3932         my $count;
3933         if ($count = $fh->sysread($buf, 8192)) {
3934             if ($buf =~ /^(.*)\(qemu\) $/s) {
3935                 $res .= $1;
3936                 last;
3937             } else {
3938                 $res .= $buf;
3939             }
3940         } else {
3941             if (!defined($count)) {
3942                 die "$!\n";
3943             }
3944             last;
3945         }
3946     }
3947
3948     die "monitor read timeout\n" if !scalar(@ready);
3949
3950     return $res;
3951 }
3952
3953 # old code, only used to shutdown old VM after update
3954 sub vm_monitor_command {
3955     my ($vmid, $cmdstr, $nocheck) = @_;
3956
3957     my $res;
3958
3959     eval {
3960         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3961
3962         my $sname = "${var_run_tmpdir}/$vmid.mon";
3963
3964         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3965             die "unable to connect to VM $vmid socket - $!\n";
3966
3967         my $timeout = 3;
3968
3969         # hack: migrate sometime blocks the monitor (when migrate_downtime
3970         # is set)
3971         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3972             $timeout = 60*60; # 1 hour
3973         }
3974
3975         # read banner;
3976         my $data = __read_avail($sock, $timeout);
3977
3978         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3979             die "got unexpected qemu monitor banner\n";
3980         }
3981
3982         my $sel = new IO::Select;
3983         $sel->add($sock);
3984
3985         if (!scalar(my @ready = $sel->can_write($timeout))) {
3986             die "monitor write error - timeout";
3987         }
3988
3989         my $fullcmd = "$cmdstr\r";
3990
3991         # syslog('info', "VM $vmid monitor command: $cmdstr");
3992
3993         my $b;
3994         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3995             die "monitor write error - $!";
3996         }
3997
3998         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3999
4000         $timeout = 20;
4001
4002         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4003             $timeout = 60*60; # 1 hour
4004         } elsif ($cmdstr =~ m/^(eject|change)/) {
4005             $timeout = 60; # note: cdrom mount command is slow
4006         }
4007         if ($res = __read_avail($sock, $timeout)) {
4008
4009             my @lines = split("\r?\n", $res);
4010
4011             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4012
4013             $res = join("\n", @lines);
4014             $res .= "\n";
4015         }
4016     };
4017
4018     my $err = $@;
4019
4020     if ($err) {
4021         syslog("err", "VM $vmid monitor command failed - $err");
4022         die $err;
4023     }
4024
4025     return $res;
4026 }
4027
4028 sub qemu_block_resize {
4029     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4030
4031     my $running = check_running($vmid);
4032
4033     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4034
4035     return if !$running;
4036
4037     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4038
4039 }
4040
4041 sub qemu_volume_snapshot {
4042     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4043
4044     my $running = check_running($vmid);
4045
4046     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4047         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4048     } else {
4049         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4050     }
4051 }
4052
4053 sub qemu_volume_snapshot_delete {
4054     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4055
4056     my $running = check_running($vmid);
4057
4058     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4059         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4060     } else {
4061         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4062     }
4063 }
4064
4065 sub set_migration_caps {
4066     my ($vmid) = @_;
4067
4068     my $cap_ref = [];
4069
4070     my $enabled_cap = {
4071         "auto-converge" => 1,
4072         "xbzrle" => 1,
4073         "x-rdma-pin-all" => 0,
4074         "zero-blocks" => 0,
4075         "compress" => 0
4076     };
4077
4078     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4079
4080     for my $supported_capability (@$supported_capabilities) {
4081         push @$cap_ref, {
4082             capability => $supported_capability->{capability},
4083             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4084         };
4085     }
4086
4087     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4088 }
4089
4090 my $fast_plug_option = {
4091     'lock' => 1,
4092     'name' => 1,
4093     'onboot' => 1,
4094     'shares' => 1,
4095     'startup' => 1,
4096     'description' => 1,
4097     'protection' => 1,
4098     'vmstatestorage' => 1,
4099 };
4100
4101 # hotplug changes in [PENDING]
4102 # $selection hash can be used to only apply specified options, for
4103 # example: { cores => 1 } (only apply changed 'cores')
4104 # $errors ref is used to return error messages
4105 sub vmconfig_hotplug_pending {
4106     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4107
4108     my $defaults = load_defaults();
4109
4110     # commit values which do not have any impact on running VM first
4111     # Note: those option cannot raise errors, we we do not care about
4112     # $selection and always apply them.
4113
4114     my $add_error = sub {
4115         my ($opt, $msg) = @_;
4116         $errors->{$opt} = "hotplug problem - $msg";
4117     };
4118
4119     my $changes = 0;
4120     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4121         if ($fast_plug_option->{$opt}) {
4122             $conf->{$opt} = $conf->{pending}->{$opt};
4123             delete $conf->{pending}->{$opt};
4124             $changes = 1;
4125         }
4126     }
4127
4128     if ($changes) {
4129         PVE::QemuConfig->write_config($vmid, $conf);
4130         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4131     }
4132
4133     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4134
4135     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4136     while (my ($opt, $force) = each %$pending_delete_hash) {
4137         next if $selection && !$selection->{$opt};
4138         eval {
4139             if ($opt eq 'hotplug') {
4140                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4141             } elsif ($opt eq 'tablet') {
4142                 die "skip\n" if !$hotplug_features->{usb};
4143                 if ($defaults->{tablet}) {
4144                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4145                 } else {
4146                     vm_deviceunplug($vmid, $conf, $opt);
4147                 }
4148             } elsif ($opt =~ m/^usb\d+/) {
4149                 die "skip\n";
4150                 # since we cannot reliably hot unplug usb devices
4151                 # we are disabling it
4152                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4153                 vm_deviceunplug($vmid, $conf, $opt);
4154             } elsif ($opt eq 'vcpus') {
4155                 die "skip\n" if !$hotplug_features->{cpu};
4156                 qemu_cpu_hotplug($vmid, $conf, undef);
4157             } elsif ($opt eq 'balloon') {
4158                 # enable balloon device is not hotpluggable
4159                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4160             } elsif ($fast_plug_option->{$opt}) {
4161                 # do nothing
4162             } elsif ($opt =~ m/^net(\d+)$/) {
4163                 die "skip\n" if !$hotplug_features->{network};
4164                 vm_deviceunplug($vmid, $conf, $opt);
4165             } elsif (is_valid_drivename($opt)) {
4166                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4167                 vm_deviceunplug($vmid, $conf, $opt);
4168                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4169             } elsif ($opt =~ m/^memory$/) {
4170                 die "skip\n" if !$hotplug_features->{memory};
4171                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4172             } elsif ($opt eq 'cpuunits') {
4173                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4174             } elsif ($opt eq 'cpulimit') {
4175                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4176             } else {
4177                 die "skip\n";
4178             }
4179         };
4180         if (my $err = $@) {
4181             &$add_error($opt, $err) if $err ne "skip\n";
4182         } else {
4183             # save new config if hotplug was successful
4184             delete $conf->{$opt};
4185             vmconfig_undelete_pending_option($conf, $opt);
4186             PVE::QemuConfig->write_config($vmid, $conf);
4187             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4188         }
4189     }
4190
4191     foreach my $opt (keys %{$conf->{pending}}) {
4192         next if $selection && !$selection->{$opt};
4193         my $value = $conf->{pending}->{$opt};
4194         eval {
4195             if ($opt eq 'hotplug') {
4196                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4197             } elsif ($opt eq 'tablet') {
4198                 die "skip\n" if !$hotplug_features->{usb};
4199                 if ($value == 1) {
4200                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4201                 } elsif ($value == 0) {
4202                     vm_deviceunplug($vmid, $conf, $opt);
4203                 }
4204             } elsif ($opt =~ m/^usb\d+$/) {
4205                 die "skip\n";
4206                 # since we cannot reliably hot unplug usb devices
4207                 # we are disabling it
4208                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4209                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4210                 die "skip\n" if !$d;
4211                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4212             } elsif ($opt eq 'vcpus') {
4213                 die "skip\n" if !$hotplug_features->{cpu};
4214                 qemu_cpu_hotplug($vmid, $conf, $value);
4215             } elsif ($opt eq 'balloon') {
4216                 # enable/disable balloning device is not hotpluggable
4217                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4218                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4219                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4220
4221                 # allow manual ballooning if shares is set to zero
4222                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4223                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4224                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4225                 }
4226             } elsif ($opt =~ m/^net(\d+)$/) {
4227                 # some changes can be done without hotplug
4228                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4229                                     $vmid, $opt, $value);
4230             } elsif (is_valid_drivename($opt)) {
4231                 # some changes can be done without hotplug
4232                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4233                                      $vmid, $opt, $value, 1);
4234             } elsif ($opt =~ m/^memory$/) { #dimms
4235                 die "skip\n" if !$hotplug_features->{memory};
4236                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4237             } elsif ($opt eq 'cpuunits') {
4238                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4239             } elsif ($opt eq 'cpulimit') {
4240                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4241                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4242             } else {
4243                 die "skip\n";  # skip non-hot-pluggable options
4244             }
4245         };
4246         if (my $err = $@) {
4247             &$add_error($opt, $err) if $err ne "skip\n";
4248         } else {
4249             # save new config if hotplug was successful
4250             $conf->{$opt} = $value;
4251             delete $conf->{pending}->{$opt};
4252             PVE::QemuConfig->write_config($vmid, $conf);
4253             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4254         }
4255     }
4256 }
4257
4258 sub try_deallocate_drive {
4259     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4260
4261     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4262         my $volid = $drive->{file};
4263         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4264             my $sid = PVE::Storage::parse_volume_id($volid);
4265             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4266
4267             # check if the disk is really unused
4268             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4269                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4270             PVE::Storage::vdisk_free($storecfg, $volid);
4271             return 1;
4272         } else {
4273             # If vm is not owner of this disk remove from config
4274             return 1;
4275         }
4276     }
4277
4278     return undef;
4279 }
4280
4281 sub vmconfig_delete_or_detach_drive {
4282     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4283
4284     my $drive = parse_drive($opt, $conf->{$opt});
4285
4286     my $rpcenv = PVE::RPCEnvironment::get();
4287     my $authuser = $rpcenv->get_user();
4288
4289     if ($force) {
4290         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4291         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4292     } else {
4293         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4294     }
4295 }
4296
4297 sub vmconfig_apply_pending {
4298     my ($vmid, $conf, $storecfg) = @_;
4299
4300     # cold plug
4301
4302     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4303     while (my ($opt, $force) = each %$pending_delete_hash) {
4304         die "internal error" if $opt =~ m/^unused/;
4305         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4306         if (!defined($conf->{$opt})) {
4307             vmconfig_undelete_pending_option($conf, $opt);
4308             PVE::QemuConfig->write_config($vmid, $conf);
4309         } elsif (is_valid_drivename($opt)) {
4310             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4311             vmconfig_undelete_pending_option($conf, $opt);
4312             delete $conf->{$opt};
4313             PVE::QemuConfig->write_config($vmid, $conf);
4314         } else {
4315             vmconfig_undelete_pending_option($conf, $opt);
4316             delete $conf->{$opt};
4317             PVE::QemuConfig->write_config($vmid, $conf);
4318         }
4319     }
4320
4321     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4322
4323     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4324         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4325
4326         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4327             # skip if nothing changed
4328         } elsif (is_valid_drivename($opt)) {
4329             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4330                 if defined($conf->{$opt});
4331             $conf->{$opt} = $conf->{pending}->{$opt};
4332         } else {
4333             $conf->{$opt} = $conf->{pending}->{$opt};
4334         }
4335
4336         delete $conf->{pending}->{$opt};
4337         PVE::QemuConfig->write_config($vmid, $conf);
4338     }
4339 }
4340
4341 my $safe_num_ne = sub {
4342     my ($a, $b) = @_;
4343
4344     return 0 if !defined($a) && !defined($b);
4345     return 1 if !defined($a);
4346     return 1 if !defined($b);
4347
4348     return $a != $b;
4349 };
4350
4351 my $safe_string_ne = sub {
4352     my ($a, $b) = @_;
4353
4354     return 0 if !defined($a) && !defined($b);
4355     return 1 if !defined($a);
4356     return 1 if !defined($b);
4357
4358     return $a ne $b;
4359 };
4360
4361 sub vmconfig_update_net {
4362     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4363
4364     my $newnet = parse_net($value);
4365
4366     if ($conf->{$opt}) {
4367         my $oldnet = parse_net($conf->{$opt});
4368
4369         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4370             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4371             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4372             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4373
4374             # for non online change, we try to hot-unplug
4375             die "skip\n" if !$hotplug;
4376             vm_deviceunplug($vmid, $conf, $opt);
4377         } else {
4378
4379             die "internal error" if $opt !~ m/net(\d+)/;
4380             my $iface = "tap${vmid}i$1";
4381
4382             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4383                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4384                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4385                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4386                 PVE::Network::tap_unplug($iface);
4387                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4388             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4389                 # Rate can be applied on its own but any change above needs to
4390                 # include the rate in tap_plug since OVS resets everything.
4391                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4392             }
4393
4394             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4395                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4396             }
4397
4398             return 1;
4399         }
4400     }
4401
4402     if ($hotplug) {
4403         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4404     } else {
4405         die "skip\n";
4406     }
4407 }
4408
4409 sub vmconfig_update_disk {
4410     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4411
4412     # fixme: do we need force?
4413
4414     my $drive = parse_drive($opt, $value);
4415
4416     if ($conf->{$opt}) {
4417
4418         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4419
4420             my $media = $drive->{media} || 'disk';
4421             my $oldmedia = $old_drive->{media} || 'disk';
4422             die "unable to change media type\n" if $media ne $oldmedia;
4423
4424             if (!drive_is_cdrom($old_drive)) {
4425
4426                 if ($drive->{file} ne $old_drive->{file}) {
4427
4428                     die "skip\n" if !$hotplug;
4429
4430                     # unplug and register as unused
4431                     vm_deviceunplug($vmid, $conf, $opt);
4432                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4433
4434                 } else {
4435                     # update existing disk
4436
4437                     # skip non hotpluggable value
4438                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4439                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4440                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4441                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4442                         die "skip\n";
4443                     }
4444
4445                     # apply throttle
4446                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4447                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4448                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4449                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4450                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4451                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4452                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4453                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4454                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4455                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4456                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4457                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4458                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4459                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4460                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4461                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4462                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4463                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4464
4465                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4466                                                    ($drive->{mbps} || 0)*1024*1024,
4467                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4468                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4469                                                    $drive->{iops} || 0,
4470                                                    $drive->{iops_rd} || 0,
4471                                                    $drive->{iops_wr} || 0,
4472                                                    ($drive->{mbps_max} || 0)*1024*1024,
4473                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4474                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4475                                                    $drive->{iops_max} || 0,
4476                                                    $drive->{iops_rd_max} || 0,
4477                                                    $drive->{iops_wr_max} || 0,
4478                                                    $drive->{bps_max_length} || 1,
4479                                                    $drive->{bps_rd_max_length} || 1,
4480                                                    $drive->{bps_wr_max_length} || 1,
4481                                                    $drive->{iops_max_length} || 1,
4482                                                    $drive->{iops_rd_max_length} || 1,
4483                                                    $drive->{iops_wr_max_length} || 1);
4484
4485                     }
4486
4487                     return 1;
4488                 }
4489
4490             } else { # cdrom
4491
4492                 if ($drive->{file} eq 'none') {
4493                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4494                 } else {
4495                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4496                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4497                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4498                 }
4499
4500                 return 1;
4501             }
4502         }
4503     }
4504
4505     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4506     # hotplug new disks
4507     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4508     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4509 }
4510
4511 sub vm_start {
4512     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4513         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4514
4515     PVE::QemuConfig->lock_config($vmid, sub {
4516         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4517
4518         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4519
4520         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4521
4522         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4523
4524         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4525             vmconfig_apply_pending($vmid, $conf, $storecfg);
4526             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4527         }
4528
4529         my $defaults = load_defaults();
4530
4531         # set environment variable useful inside network script
4532         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4533
4534         my $local_volumes = {};
4535
4536         if ($targetstorage) {
4537             foreach_drive($conf, sub {
4538                 my ($ds, $drive) = @_;
4539
4540                 return if drive_is_cdrom($drive);
4541
4542                 my $volid = $drive->{file};
4543
4544                 return if !$volid;
4545
4546                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4547
4548                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4549                 return if $scfg->{shared};
4550                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4551             });
4552
4553             my $format = undef;
4554
4555             foreach my $opt (sort keys %$local_volumes) {
4556
4557                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4558                 my $drive = parse_drive($opt, $conf->{$opt});
4559
4560                 #if remote storage is specified, use default format
4561                 if ($targetstorage && $targetstorage ne "1") {
4562                     $storeid = $targetstorage;
4563                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4564                     $format = $defFormat;
4565                 } else {
4566                     #else we use same format than original
4567                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4568                     $format = qemu_img_format($scfg, $volid);
4569                 }
4570
4571                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4572                 my $newdrive = $drive;
4573                 $newdrive->{format} = $format;
4574                 $newdrive->{file} = $newvolid;
4575                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4576                 $local_volumes->{$opt} = $drivestr;
4577                 #pass drive to conf for command line
4578                 $conf->{$opt} = $drivestr;
4579             }
4580         }
4581
4582         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4583
4584         my $migrate_port = 0;
4585         my $migrate_uri;
4586         if ($statefile) {
4587             if ($statefile eq 'tcp') {
4588                 my $localip = "localhost";
4589                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4590                 my $nodename = PVE::INotify::nodename();
4591
4592                 if (!defined($migration_type)) {
4593                     if (defined($datacenterconf->{migration}->{type})) {
4594                         $migration_type = $datacenterconf->{migration}->{type};
4595                     } else {
4596                         $migration_type = 'secure';
4597                     }
4598                 }
4599
4600                 if ($migration_type eq 'insecure') {
4601                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4602                     if ($migrate_network_addr) {
4603                         $localip = $migrate_network_addr;
4604                     } else {
4605                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4606                     }
4607
4608                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4609                 }
4610
4611                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4612                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4613                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4614                 push @$cmd, '-incoming', $migrate_uri;
4615                 push @$cmd, '-S';
4616
4617             } elsif ($statefile eq 'unix') {
4618                 # should be default for secure migrations as a ssh TCP forward
4619                 # tunnel is not deterministic reliable ready and fails regurarly
4620                 # to set up in time, so use UNIX socket forwards
4621                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4622                 unlink $socket_addr;
4623
4624                 $migrate_uri = "unix:$socket_addr";
4625
4626                 push @$cmd, '-incoming', $migrate_uri;
4627                 push @$cmd, '-S';
4628
4629             } else {
4630                 push @$cmd, '-loadstate', $statefile;
4631             }
4632         } elsif ($paused) {
4633             push @$cmd, '-S';
4634         }
4635
4636         # host pci devices
4637         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4638           my $d = parse_hostpci($conf->{"hostpci$i"});
4639           next if !$d;
4640           my $pcidevices = $d->{pciid};
4641           foreach my $pcidevice (@$pcidevices) {
4642                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4643
4644                 my $info = pci_device_info("0000:$pciid");
4645                 die "IOMMU not present\n" if !check_iommu_support();
4646                 die "no pci device info for device '$pciid'\n" if !$info;
4647                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4648                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4649           }
4650         }
4651
4652         PVE::Storage::activate_volumes($storecfg, $vollist);
4653
4654         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4655             my $cmd = [];
4656             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4657             eval  { run_command($cmd); };
4658         }
4659
4660         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4661                                                   : $defaults->{cpuunits};
4662
4663         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4664
4665         my %properties = (
4666             Slice => 'qemu.slice',
4667             KillMode => 'none',
4668             CPUShares => $cpuunits
4669         );
4670
4671         if (my $cpulimit = $conf->{cpulimit}) {
4672             $properties{CPUQuota} = int($cpulimit * 100);
4673         }
4674         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4675
4676         if ($conf->{hugepages}) {
4677
4678             my $code = sub {
4679                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4680                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4681
4682                 PVE::QemuServer::Memory::hugepages_mount();
4683                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4684
4685                 eval  {
4686                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4687                     run_command($cmd, %run_params);
4688                 };
4689
4690                 if (my $err = $@) {
4691                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4692                     die $err;
4693                 }
4694
4695                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4696             };
4697             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4698
4699         } else {
4700             eval  {
4701                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4702                 run_command($cmd, %run_params);
4703             };
4704         }
4705
4706         if (my $err = $@) {
4707             # deactivate volumes if start fails
4708             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4709             die "start failed: $err";
4710         }
4711
4712         print "migration listens on $migrate_uri\n" if $migrate_uri;
4713
4714         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4715             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4716             warn $@ if $@;
4717         }
4718
4719         #start nbd server for storage migration
4720         if ($targetstorage) {
4721             my $nodename = PVE::INotify::nodename();
4722             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4723             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4724             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4725             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4726
4727             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4728
4729             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4730
4731             foreach my $opt (sort keys %$local_volumes) {
4732                 my $volid = $local_volumes->{$opt};
4733                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4734                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4735                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4736             }
4737         }
4738
4739         if ($migratedfrom) {
4740             eval {
4741                 set_migration_caps($vmid);
4742             };
4743             warn $@ if $@;
4744
4745             if ($spice_port) {
4746                 print "spice listens on port $spice_port\n";
4747                 if ($spice_ticket) {
4748                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4749                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4750                 }
4751             }
4752
4753         } else {
4754             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4755                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4756                     if $conf->{balloon};
4757             }
4758
4759             foreach my $opt (keys %$conf) {
4760                 next if $opt !~  m/^net\d+$/;
4761                 my $nicconf = parse_net($conf->{$opt});
4762                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4763             }
4764         }
4765
4766         vm_mon_cmd_nocheck($vmid, 'qom-set',
4767                     path => "machine/peripheral/balloon0",
4768                     property => "guest-stats-polling-interval",
4769                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4770
4771     });
4772 }
4773
4774 sub vm_mon_cmd {
4775     my ($vmid, $execute, %params) = @_;
4776
4777     my $cmd = { execute => $execute, arguments => \%params };
4778     vm_qmp_command($vmid, $cmd);
4779 }
4780
4781 sub vm_mon_cmd_nocheck {
4782     my ($vmid, $execute, %params) = @_;
4783
4784     my $cmd = { execute => $execute, arguments => \%params };
4785     vm_qmp_command($vmid, $cmd, 1);
4786 }
4787
4788 sub vm_qmp_command {
4789     my ($vmid, $cmd, $nocheck) = @_;
4790
4791     my $res;
4792
4793     my $timeout;
4794     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4795         $timeout = $cmd->{arguments}->{timeout};
4796         delete $cmd->{arguments}->{timeout};
4797     }
4798
4799     eval {
4800         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4801         my $sname = qmp_socket($vmid);
4802         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4803             my $qmpclient = PVE::QMPClient->new();
4804
4805             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4806         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4807             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4808                 if scalar(%{$cmd->{arguments}});
4809             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4810         } else {
4811             die "unable to open monitor socket\n";
4812         }
4813     };
4814     if (my $err = $@) {
4815         syslog("err", "VM $vmid qmp command failed - $err");
4816         die $err;
4817     }
4818
4819     return $res;
4820 }
4821
4822 sub vm_human_monitor_command {
4823     my ($vmid, $cmdline) = @_;
4824
4825     my $res;
4826
4827     my $cmd = {
4828         execute => 'human-monitor-command',
4829         arguments => { 'command-line' => $cmdline},
4830     };
4831
4832     return vm_qmp_command($vmid, $cmd);
4833 }
4834
4835 sub vm_commandline {
4836     my ($storecfg, $vmid) = @_;
4837
4838     my $conf = PVE::QemuConfig->load_config($vmid);
4839
4840     my $defaults = load_defaults();
4841
4842     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4843
4844     return PVE::Tools::cmd2string($cmd);
4845 }
4846
4847 sub vm_reset {
4848     my ($vmid, $skiplock) = @_;
4849
4850     PVE::QemuConfig->lock_config($vmid, sub {
4851
4852         my $conf = PVE::QemuConfig->load_config($vmid);
4853
4854         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4855
4856         vm_mon_cmd($vmid, "system_reset");
4857     });
4858 }
4859
4860 sub get_vm_volumes {
4861     my ($conf) = @_;
4862
4863     my $vollist = [];
4864     foreach_volid($conf, sub {
4865         my ($volid, $attr) = @_;
4866
4867         return if $volid =~ m|^/|;
4868
4869         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4870         return if !$sid;
4871
4872         push @$vollist, $volid;
4873     });
4874
4875     return $vollist;
4876 }
4877
4878 sub vm_stop_cleanup {
4879     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4880
4881     eval {
4882
4883         if (!$keepActive) {
4884             my $vollist = get_vm_volumes($conf);
4885             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4886         }
4887
4888         foreach my $ext (qw(mon qmp pid vnc qga)) {
4889             unlink "/var/run/qemu-server/${vmid}.$ext";
4890         }
4891
4892         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4893     };
4894     warn $@ if $@; # avoid errors - just warn
4895 }
4896
4897 # Note: use $nockeck to skip tests if VM configuration file exists.
4898 # We need that when migration VMs to other nodes (files already moved)
4899 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4900 sub vm_stop {
4901     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4902
4903     $force = 1 if !defined($force) && !$shutdown;
4904
4905     if ($migratedfrom){
4906         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4907         kill 15, $pid if $pid;
4908         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4909         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4910         return;
4911     }
4912
4913     PVE::QemuConfig->lock_config($vmid, sub {
4914
4915         my $pid = check_running($vmid, $nocheck);
4916         return if !$pid;
4917
4918         my $conf;
4919         if (!$nocheck) {
4920             $conf = PVE::QemuConfig->load_config($vmid);
4921             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4922             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4923                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4924                 $timeout = $opts->{down} if $opts->{down};
4925             }
4926         }
4927
4928         $timeout = 60 if !defined($timeout);
4929
4930         eval {
4931             if ($shutdown) {
4932                 if (defined($conf) && $conf->{agent}) {
4933                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4934                 } else {
4935                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4936                 }
4937             } else {
4938                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4939             }
4940         };
4941         my $err = $@;
4942
4943         if (!$err) {
4944             my $count = 0;
4945             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4946                 $count++;
4947                 sleep 1;
4948             }
4949
4950             if ($count >= $timeout) {
4951                 if ($force) {
4952                     warn "VM still running - terminating now with SIGTERM\n";
4953                     kill 15, $pid;
4954                 } else {
4955                     die "VM quit/powerdown failed - got timeout\n";
4956                 }
4957             } else {
4958                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4959                 return;
4960             }
4961         } else {
4962             if ($force) {
4963                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4964                 kill 15, $pid;
4965             } else {
4966                 die "VM quit/powerdown failed\n";
4967             }
4968         }
4969
4970         # wait again
4971         $timeout = 10;
4972
4973         my $count = 0;
4974         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4975             $count++;
4976             sleep 1;
4977         }
4978
4979         if ($count >= $timeout) {
4980             warn "VM still running - terminating now with SIGKILL\n";
4981             kill 9, $pid;
4982             sleep 1;
4983         }
4984
4985         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4986    });
4987 }
4988
4989 sub vm_suspend {
4990     my ($vmid, $skiplock) = @_;
4991
4992     PVE::QemuConfig->lock_config($vmid, sub {
4993
4994         my $conf = PVE::QemuConfig->load_config($vmid);
4995
4996         PVE::QemuConfig->check_lock($conf)
4997             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4998
4999         vm_mon_cmd($vmid, "stop");
5000     });
5001 }
5002
5003 sub vm_resume {
5004     my ($vmid, $skiplock, $nocheck) = @_;
5005
5006     PVE::QemuConfig->lock_config($vmid, sub {
5007
5008         if (!$nocheck) {
5009
5010             my $conf = PVE::QemuConfig->load_config($vmid);
5011
5012             PVE::QemuConfig->check_lock($conf)
5013                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5014
5015             vm_mon_cmd($vmid, "cont");
5016
5017         } else {
5018             vm_mon_cmd_nocheck($vmid, "cont");
5019         }
5020     });
5021 }
5022
5023 sub vm_sendkey {
5024     my ($vmid, $skiplock, $key) = @_;
5025
5026     PVE::QemuConfig->lock_config($vmid, sub {
5027
5028         my $conf = PVE::QemuConfig->load_config($vmid);
5029
5030         # there is no qmp command, so we use the human monitor command
5031         vm_human_monitor_command($vmid, "sendkey $key");
5032     });
5033 }
5034
5035 sub vm_destroy {
5036     my ($storecfg, $vmid, $skiplock) = @_;
5037
5038     PVE::QemuConfig->lock_config($vmid, sub {
5039
5040         my $conf = PVE::QemuConfig->load_config($vmid);
5041
5042         if (!check_running($vmid)) {
5043             destroy_vm($storecfg, $vmid, undef, $skiplock);
5044         } else {
5045             die "VM $vmid is running - destroy failed\n";
5046         }
5047     });
5048 }
5049
5050 # pci helpers
5051
5052 sub file_write {
5053     my ($filename, $buf) = @_;
5054
5055     my $fh = IO::File->new($filename, "w");
5056     return undef if !$fh;
5057
5058     my $res = print $fh $buf;
5059
5060     $fh->close();
5061
5062     return $res;
5063 }
5064
5065 sub pci_device_info {
5066     my ($name) = @_;
5067
5068     my $res;
5069
5070     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5071     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5072
5073     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5074     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5075
5076     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5077     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5078
5079     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5080     return undef if !defined($product) || $product !~ s/^0x//;
5081
5082     $res = {
5083         name => $name,
5084         vendor => $vendor,
5085         product => $product,
5086         domain => $domain,
5087         bus => $bus,
5088         slot => $slot,
5089         func => $func,
5090         irq => $irq,
5091         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5092     };
5093
5094     return $res;
5095 }
5096
5097 sub pci_dev_reset {
5098     my ($dev) = @_;
5099
5100     my $name = $dev->{name};
5101
5102     my $fn = "$pcisysfs/devices/$name/reset";
5103
5104     return file_write($fn, "1");
5105 }
5106
5107 sub pci_dev_bind_to_vfio {
5108     my ($dev) = @_;
5109
5110     my $name = $dev->{name};
5111
5112     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5113
5114     if (!-d $vfio_basedir) {
5115         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5116     }
5117     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5118
5119     my $testdir = "$vfio_basedir/$name";
5120     return 1 if -d $testdir;
5121
5122     my $data = "$dev->{vendor} $dev->{product}";
5123     return undef if !file_write("$vfio_basedir/new_id", $data);
5124
5125     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5126     if (!file_write($fn, $name)) {
5127         return undef if -f $fn;
5128     }
5129
5130     $fn = "$vfio_basedir/bind";
5131     if (! -d $testdir) {
5132         return undef if !file_write($fn, $name);
5133     }
5134
5135     return -d $testdir;
5136 }
5137
5138 sub pci_dev_group_bind_to_vfio {
5139     my ($pciid) = @_;
5140
5141     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5142
5143     if (!-d $vfio_basedir) {
5144         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5145     }
5146     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5147
5148     # get IOMMU group devices
5149     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5150       my @devs = grep /^0000:/, readdir($D);
5151     closedir($D);
5152
5153     foreach my $pciid (@devs) {
5154         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5155
5156         # pci bridges, switches or root ports are not supported
5157         # they have a pci_bus subdirectory so skip them
5158         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5159
5160         my $info = pci_device_info($1);
5161         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5162     }
5163
5164     return 1;
5165 }
5166
5167 # vzdump restore implementaion
5168
5169 sub tar_archive_read_firstfile {
5170     my $archive = shift;
5171
5172     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5173
5174     # try to detect archive type first
5175     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5176         die "unable to open file '$archive'\n";
5177     my $firstfile = <$fh>;
5178     kill 15, $pid;
5179     close $fh;
5180
5181     die "ERROR: archive contaions no data\n" if !$firstfile;
5182     chomp $firstfile;
5183
5184     return $firstfile;
5185 }
5186
5187 sub tar_restore_cleanup {
5188     my ($storecfg, $statfile) = @_;
5189
5190     print STDERR "starting cleanup\n";
5191
5192     if (my $fd = IO::File->new($statfile, "r")) {
5193         while (defined(my $line = <$fd>)) {
5194             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5195                 my $volid = $2;
5196                 eval {
5197                     if ($volid =~ m|^/|) {
5198                         unlink $volid || die 'unlink failed\n';
5199                     } else {
5200                         PVE::Storage::vdisk_free($storecfg, $volid);
5201                     }
5202                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5203                 };
5204                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5205             } else {
5206                 print STDERR "unable to parse line in statfile - $line";
5207             }
5208         }
5209         $fd->close();
5210     }
5211 }
5212
5213 sub restore_archive {
5214     my ($archive, $vmid, $user, $opts) = @_;
5215
5216     my $format = $opts->{format};
5217     my $comp;
5218
5219     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5220         $format = 'tar' if !$format;
5221         $comp = 'gzip';
5222     } elsif ($archive =~ m/\.tar$/) {
5223         $format = 'tar' if !$format;
5224     } elsif ($archive =~ m/.tar.lzo$/) {
5225         $format = 'tar' if !$format;
5226         $comp = 'lzop';
5227     } elsif ($archive =~ m/\.vma$/) {
5228         $format = 'vma' if !$format;
5229     } elsif ($archive =~ m/\.vma\.gz$/) {
5230         $format = 'vma' if !$format;
5231         $comp = 'gzip';
5232     } elsif ($archive =~ m/\.vma\.lzo$/) {
5233         $format = 'vma' if !$format;
5234         $comp = 'lzop';
5235     } else {
5236         $format = 'vma' if !$format; # default
5237     }
5238
5239     # try to detect archive format
5240     if ($format eq 'tar') {
5241         return restore_tar_archive($archive, $vmid, $user, $opts);
5242     } else {
5243         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5244     }
5245 }
5246
5247 sub restore_update_config_line {
5248     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5249
5250     return if $line =~ m/^\#qmdump\#/;
5251     return if $line =~ m/^\#vzdump\#/;
5252     return if $line =~ m/^lock:/;
5253     return if $line =~ m/^unused\d+:/;
5254     return if $line =~ m/^parent:/;
5255     return if $line =~ m/^template:/; # restored VM is never a template
5256
5257     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5258     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5259         # try to convert old 1.X settings
5260         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5261         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5262             my ($model, $macaddr) = split(/\=/, $devconfig);
5263             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5264             my $net = {
5265                 model => $model,
5266                 bridge => "vmbr$ind",
5267                 macaddr => $macaddr,
5268             };
5269             my $netstr = print_net($net);
5270
5271             print $outfd "net$cookie->{netcount}: $netstr\n";
5272             $cookie->{netcount}++;
5273         }
5274     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5275         my ($id, $netstr) = ($1, $2);
5276         my $net = parse_net($netstr);
5277         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5278         $netstr = print_net($net);
5279         print $outfd "$id: $netstr\n";
5280     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5281         my $virtdev = $1;
5282         my $value = $3;
5283         my $di = parse_drive($virtdev, $value);
5284         if (defined($di->{backup}) && !$di->{backup}) {
5285             print $outfd "#$line";
5286         } elsif ($map->{$virtdev}) {
5287             delete $di->{format}; # format can change on restore
5288             $di->{file} = $map->{$virtdev};
5289             $value = print_drive($vmid, $di);
5290             print $outfd "$virtdev: $value\n";
5291         } else {
5292             print $outfd $line;
5293         }
5294     } else {
5295         print $outfd $line;
5296     }
5297 }
5298
5299 sub scan_volids {
5300     my ($cfg, $vmid) = @_;
5301
5302     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5303
5304     my $volid_hash = {};
5305     foreach my $storeid (keys %$info) {
5306         foreach my $item (@{$info->{$storeid}}) {
5307             next if !($item->{volid} && $item->{size});
5308             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5309             $volid_hash->{$item->{volid}} = $item;
5310         }
5311     }
5312
5313     return $volid_hash;
5314 }
5315
5316 sub is_volume_in_use {
5317     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5318
5319     my $path = PVE::Storage::path($storecfg, $volid);
5320
5321     my $scan_config = sub {
5322         my ($cref, $snapname) = @_;
5323
5324         foreach my $key (keys %$cref) {
5325             my $value = $cref->{$key};
5326             if (is_valid_drivename($key)) {
5327                 next if $skip_drive && $key eq $skip_drive;
5328                 my $drive = parse_drive($key, $value);
5329                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5330                 return 1 if $volid eq $drive->{file};
5331                 if ($drive->{file} =~ m!^/!) {
5332                     return 1 if $drive->{file} eq $path;
5333                 } else {
5334                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5335                     next if !$storeid;
5336                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5337                     next if !$scfg;
5338                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5339                 }
5340             }
5341         }
5342
5343         return 0;
5344     };
5345
5346     return 1 if &$scan_config($conf);
5347
5348     undef $skip_drive;
5349
5350     foreach my $snapname (keys %{$conf->{snapshots}}) {
5351         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5352     }
5353
5354     return 0;
5355 }
5356
5357 sub update_disksize {
5358     my ($vmid, $conf, $volid_hash) = @_;
5359
5360     my $changes;
5361
5362     my $used = {};
5363
5364     # Note: it is allowed to define multiple storages with same path (alias), so
5365     # we need to check both 'volid' and real 'path' (two different volid can point
5366     # to the same path).
5367
5368     my $usedpath = {};
5369
5370     # update size info
5371     foreach my $opt (keys %$conf) {
5372         if (is_valid_drivename($opt)) {
5373             my $drive = parse_drive($opt, $conf->{$opt});
5374             my $volid = $drive->{file};
5375             next if !$volid;
5376
5377             $used->{$volid} = 1;
5378             if ($volid_hash->{$volid} &&
5379                 (my $path = $volid_hash->{$volid}->{path})) {
5380                 $usedpath->{$path} = 1;
5381             }
5382
5383             next if drive_is_cdrom($drive);
5384             next if !$volid_hash->{$volid};
5385
5386             $drive->{size} = $volid_hash->{$volid}->{size};
5387             my $new = print_drive($vmid, $drive);
5388             if ($new ne $conf->{$opt}) {
5389                 $changes = 1;
5390                 $conf->{$opt} = $new;
5391             }
5392         }
5393     }
5394
5395     # remove 'unusedX' entry if volume is used
5396     foreach my $opt (keys %$conf) {
5397         next if $opt !~ m/^unused\d+$/;
5398         my $volid = $conf->{$opt};
5399         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5400         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5401             $changes = 1;
5402             delete $conf->{$opt};
5403         }
5404     }
5405
5406     foreach my $volid (sort keys %$volid_hash) {
5407         next if $volid =~ m/vm-$vmid-state-/;
5408         next if $used->{$volid};
5409         my $path = $volid_hash->{$volid}->{path};
5410         next if !$path; # just to be sure
5411         next if $usedpath->{$path};
5412         $changes = 1;
5413         PVE::QemuConfig->add_unused_volume($conf, $volid);
5414         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5415     }
5416
5417     return $changes;
5418 }
5419
5420 sub rescan {
5421     my ($vmid, $nolock) = @_;
5422
5423     my $cfg = PVE::Storage::config();
5424
5425     my $volid_hash = scan_volids($cfg, $vmid);
5426
5427     my $updatefn =  sub {
5428         my ($vmid) = @_;
5429
5430         my $conf = PVE::QemuConfig->load_config($vmid);
5431
5432         PVE::QemuConfig->check_lock($conf);
5433
5434         my $vm_volids = {};
5435         foreach my $volid (keys %$volid_hash) {
5436             my $info = $volid_hash->{$volid};
5437             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5438         }
5439
5440         my $changes = update_disksize($vmid, $conf, $vm_volids);
5441
5442         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5443     };
5444
5445     if (defined($vmid)) {
5446         if ($nolock) {
5447             &$updatefn($vmid);
5448         } else {
5449             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5450         }
5451     } else {
5452         my $vmlist = config_list();
5453         foreach my $vmid (keys %$vmlist) {
5454             if ($nolock) {
5455                 &$updatefn($vmid);
5456             } else {
5457                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5458             }
5459         }
5460     }
5461 }
5462
5463 sub restore_vma_archive {
5464     my ($archive, $vmid, $user, $opts, $comp) = @_;
5465
5466     my $input = $archive eq '-' ? "<&STDIN" : undef;
5467     my $readfrom = $archive;
5468
5469     my $uncomp = '';
5470     if ($comp) {
5471         $readfrom = '-';
5472         my $qarchive = PVE::Tools::shellquote($archive);
5473         if ($comp eq 'gzip') {
5474             $uncomp = "zcat $qarchive|";
5475         } elsif ($comp eq 'lzop') {
5476             $uncomp = "lzop -d -c $qarchive|";
5477         } else {
5478             die "unknown compression method '$comp'\n";
5479         }
5480
5481     }
5482
5483     my $tmpdir = "/var/tmp/vzdumptmp$$";
5484     rmtree $tmpdir;
5485
5486     # disable interrupts (always do cleanups)
5487     local $SIG{INT} =
5488         local $SIG{TERM} =
5489         local $SIG{QUIT} =
5490         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5491
5492     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5493     POSIX::mkfifo($mapfifo, 0600);
5494     my $fifofh;
5495
5496     my $openfifo = sub {
5497         open($fifofh, '>', $mapfifo) || die $!;
5498     };
5499
5500     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5501
5502     my $oldtimeout;
5503     my $timeout = 5;
5504
5505     my $devinfo = {};
5506
5507     my $rpcenv = PVE::RPCEnvironment::get();
5508
5509     my $conffile = PVE::QemuConfig->config_file($vmid);
5510     my $tmpfn = "$conffile.$$.tmp";
5511
5512     # Note: $oldconf is undef if VM does not exists
5513     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5514     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5515
5516     my $print_devmap = sub {
5517         my $virtdev_hash = {};
5518
5519         my $cfgfn = "$tmpdir/qemu-server.conf";
5520
5521         # we can read the config - that is already extracted
5522         my $fh = IO::File->new($cfgfn, "r") ||
5523             "unable to read qemu-server.conf - $!\n";
5524
5525         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5526         if (-f $fwcfgfn) {
5527             my $pve_firewall_dir = '/etc/pve/firewall';
5528             mkdir $pve_firewall_dir; # make sure the dir exists
5529             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5530         }
5531
5532         while (defined(my $line = <$fh>)) {
5533             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5534                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5535                 die "archive does not contain data for drive '$virtdev'\n"
5536                     if !$devinfo->{$devname};
5537                 if (defined($opts->{storage})) {
5538                     $storeid = $opts->{storage} || 'local';
5539                 } elsif (!$storeid) {
5540                     $storeid = 'local';
5541                 }
5542                 $format = 'raw' if !$format;
5543                 $devinfo->{$devname}->{devname} = $devname;
5544                 $devinfo->{$devname}->{virtdev} = $virtdev;
5545                 $devinfo->{$devname}->{format} = $format;
5546                 $devinfo->{$devname}->{storeid} = $storeid;
5547
5548                 # check permission on storage
5549                 my $pool = $opts->{pool}; # todo: do we need that?
5550                 if ($user ne 'root@pam') {
5551                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5552                 }
5553
5554                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5555             }
5556         }
5557
5558         foreach my $devname (keys %$devinfo) {
5559             die "found no device mapping information for device '$devname'\n"
5560                 if !$devinfo->{$devname}->{virtdev};
5561         }
5562
5563         my $cfg = PVE::Storage::config();
5564
5565         # create empty/temp config
5566         if ($oldconf) {
5567             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5568             foreach_drive($oldconf, sub {
5569                 my ($ds, $drive) = @_;
5570
5571                 return if drive_is_cdrom($drive);
5572
5573                 my $volid = $drive->{file};
5574
5575                 return if !$volid || $volid =~ m|^/|;
5576
5577                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5578                 return if !$path || !$owner || ($owner != $vmid);
5579
5580                 # Note: only delete disk we want to restore
5581                 # other volumes will become unused
5582                 if ($virtdev_hash->{$ds}) {
5583                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5584                     if (my $err = $@) {
5585                         warn $err;
5586                     }
5587                 }
5588             });
5589
5590             # delete vmstate files
5591             # since after the restore we have no snapshots anymore
5592             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5593                 my $snap = $oldconf->{snapshots}->{$snapname};
5594                 if ($snap->{vmstate}) {
5595                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5596                     if (my $err = $@) {
5597                         warn $err;
5598                     }
5599                 }
5600             }
5601         }
5602
5603         my $map = {};
5604         foreach my $virtdev (sort keys %$virtdev_hash) {
5605             my $d = $virtdev_hash->{$virtdev};
5606             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5607             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5608
5609             # test if requested format is supported
5610             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5611             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5612             $d->{format} = $defFormat if !$supported;
5613
5614             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5615                                                   $d->{format}, undef, $alloc_size);
5616             print STDERR "new volume ID is '$volid'\n";
5617             $d->{volid} = $volid;
5618             my $path = PVE::Storage::path($cfg, $volid);
5619
5620             PVE::Storage::activate_volumes($cfg,[$volid]);
5621
5622             my $write_zeros = 1;
5623             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5624                 $write_zeros = 0;
5625             }
5626
5627             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5628
5629             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5630             $map->{$virtdev} = $volid;
5631         }
5632
5633         $fh->seek(0, 0) || die "seek failed - $!\n";
5634
5635         my $outfd = new IO::File ($tmpfn, "w") ||
5636             die "unable to write config for VM $vmid\n";
5637
5638         my $cookie = { netcount => 0 };
5639         while (defined(my $line = <$fh>)) {
5640             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5641         }
5642
5643         $fh->close();
5644         $outfd->close();
5645     };
5646
5647     eval {
5648         # enable interrupts
5649         local $SIG{INT} =
5650             local $SIG{TERM} =
5651             local $SIG{QUIT} =
5652             local $SIG{HUP} =
5653             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5654         local $SIG{ALRM} = sub { die "got timeout\n"; };
5655
5656         $oldtimeout = alarm($timeout);
5657
5658         my $parser = sub {
5659             my $line = shift;
5660
5661             print "$line\n";
5662
5663             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5664                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5665                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5666             } elsif ($line =~ m/^CTIME: /) {
5667                 # we correctly received the vma config, so we can disable
5668                 # the timeout now for disk allocation (set to 10 minutes, so
5669                 # that we always timeout if something goes wrong)
5670                 alarm(600);
5671                 &$print_devmap();
5672                 print $fifofh "done\n";
5673                 my $tmp = $oldtimeout || 0;
5674                 $oldtimeout = undef;
5675                 alarm($tmp);
5676                 close($fifofh);
5677             }
5678         };
5679
5680         print "restore vma archive: $cmd\n";
5681         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5682     };
5683     my $err = $@;
5684
5685     alarm($oldtimeout) if $oldtimeout;
5686
5687     my $vollist = [];
5688     foreach my $devname (keys %$devinfo) {
5689         my $volid = $devinfo->{$devname}->{volid};
5690         push @$vollist, $volid if $volid;
5691     }
5692
5693     my $cfg = PVE::Storage::config();
5694     PVE::Storage::deactivate_volumes($cfg, $vollist);
5695
5696     unlink $mapfifo;
5697
5698     if ($err) {
5699         rmtree $tmpdir;
5700         unlink $tmpfn;
5701
5702         foreach my $devname (keys %$devinfo) {
5703             my $volid = $devinfo->{$devname}->{volid};
5704             next if !$volid;
5705             eval {
5706                 if ($volid =~ m|^/|) {
5707                     unlink $volid || die 'unlink failed\n';
5708                 } else {
5709                     PVE::Storage::vdisk_free($cfg, $volid);
5710                 }
5711                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5712             };
5713             print STDERR "unable to cleanup '$volid' - $@" if $@;
5714         }
5715         die $err;
5716     }
5717
5718     rmtree $tmpdir;
5719
5720     rename($tmpfn, $conffile) ||
5721         die "unable to commit configuration file '$conffile'\n";
5722
5723     PVE::Cluster::cfs_update(); # make sure we read new file
5724
5725     eval { rescan($vmid, 1); };
5726     warn $@ if $@;
5727 }
5728
5729 sub restore_tar_archive {
5730     my ($archive, $vmid, $user, $opts) = @_;
5731
5732     if ($archive ne '-') {
5733         my $firstfile = tar_archive_read_firstfile($archive);
5734         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5735             if $firstfile ne 'qemu-server.conf';
5736     }
5737
5738     my $storecfg = PVE::Storage::config();
5739
5740     # destroy existing data - keep empty config
5741     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5742     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5743
5744     my $tocmd = "/usr/lib/qemu-server/qmextract";
5745
5746     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5747     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5748     $tocmd .= ' --prealloc' if $opts->{prealloc};
5749     $tocmd .= ' --info' if $opts->{info};
5750
5751     # tar option "xf" does not autodetect compression when read from STDIN,
5752     # so we pipe to zcat
5753     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5754         PVE::Tools::shellquote("--to-command=$tocmd");
5755
5756     my $tmpdir = "/var/tmp/vzdumptmp$$";
5757     mkpath $tmpdir;
5758
5759     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5760     local $ENV{VZDUMP_VMID} = $vmid;
5761     local $ENV{VZDUMP_USER} = $user;
5762
5763     my $conffile = PVE::QemuConfig->config_file($vmid);
5764     my $tmpfn = "$conffile.$$.tmp";
5765
5766     # disable interrupts (always do cleanups)
5767     local $SIG{INT} =
5768         local $SIG{TERM} =
5769         local $SIG{QUIT} =
5770         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5771
5772     eval {
5773         # enable interrupts
5774         local $SIG{INT} =
5775             local $SIG{TERM} =
5776             local $SIG{QUIT} =
5777             local $SIG{HUP} =
5778             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5779
5780         if ($archive eq '-') {
5781             print "extracting archive from STDIN\n";
5782             run_command($cmd, input => "<&STDIN");
5783         } else {
5784             print "extracting archive '$archive'\n";
5785             run_command($cmd);
5786         }
5787
5788         return if $opts->{info};
5789
5790         # read new mapping
5791         my $map = {};
5792         my $statfile = "$tmpdir/qmrestore.stat";
5793         if (my $fd = IO::File->new($statfile, "r")) {
5794             while (defined (my $line = <$fd>)) {
5795                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5796                     $map->{$1} = $2 if $1;
5797                 } else {
5798                     print STDERR "unable to parse line in statfile - $line\n";
5799                 }
5800             }
5801             $fd->close();
5802         }
5803
5804         my $confsrc = "$tmpdir/qemu-server.conf";
5805
5806         my $srcfd = new IO::File($confsrc, "r") ||
5807             die "unable to open file '$confsrc'\n";
5808
5809         my $outfd = new IO::File ($tmpfn, "w") ||
5810             die "unable to write config for VM $vmid\n";
5811
5812         my $cookie = { netcount => 0 };
5813         while (defined (my $line = <$srcfd>)) {
5814             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5815         }
5816
5817         $srcfd->close();
5818         $outfd->close();
5819     };
5820     my $err = $@;
5821
5822     if ($err) {
5823
5824         unlink $tmpfn;
5825
5826         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5827
5828         die $err;
5829     }
5830
5831     rmtree $tmpdir;
5832
5833     rename $tmpfn, $conffile ||
5834         die "unable to commit configuration file '$conffile'\n";
5835
5836     PVE::Cluster::cfs_update(); # make sure we read new file
5837
5838     eval { rescan($vmid, 1); };
5839     warn $@ if $@;
5840 };
5841
5842 sub foreach_storage_used_by_vm {
5843     my ($conf, $func) = @_;
5844
5845     my $sidhash = {};
5846
5847     foreach_drive($conf, sub {
5848         my ($ds, $drive) = @_;
5849         return if drive_is_cdrom($drive);
5850
5851         my $volid = $drive->{file};
5852
5853         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5854         $sidhash->{$sid} = $sid if $sid;
5855     });
5856
5857     foreach my $sid (sort keys %$sidhash) {
5858         &$func($sid);
5859     }
5860 }
5861
5862 sub do_snapshots_with_qemu {
5863     my ($storecfg, $volid) = @_;
5864
5865     my $storage_name = PVE::Storage::parse_volume_id($volid);
5866
5867     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5868         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5869         return 1;
5870     }
5871
5872     if ($volid =~ m/\.(qcow2|qed)$/){
5873         return 1;
5874     }
5875
5876     return undef;
5877 }
5878
5879 sub qga_check_running {
5880     my ($vmid) = @_;
5881
5882     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5883     if ($@) {
5884         warn "Qemu Guest Agent are not running - $@";
5885         return 0;
5886     }
5887     return 1;
5888 }
5889
5890 sub template_create {
5891     my ($vmid, $conf, $disk) = @_;
5892
5893     my $storecfg = PVE::Storage::config();
5894
5895     foreach_drive($conf, sub {
5896         my ($ds, $drive) = @_;
5897
5898         return if drive_is_cdrom($drive);
5899         return if $disk && $ds ne $disk;
5900
5901         my $volid = $drive->{file};
5902         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5903
5904         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5905         $drive->{file} = $voliddst;
5906         $conf->{$ds} = print_drive($vmid, $drive);
5907         PVE::QemuConfig->write_config($vmid, $conf);
5908     });
5909 }
5910
5911 sub qemu_img_convert {
5912     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5913
5914     my $storecfg = PVE::Storage::config();
5915     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5916     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5917
5918     if ($src_storeid && $dst_storeid) {
5919
5920         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5921
5922         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5923         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5924
5925         my $src_format = qemu_img_format($src_scfg, $src_volname);
5926         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5927
5928         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5929         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5930
5931         my $cmd = [];
5932         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5933         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5934         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5935         if ($is_zero_initialized) {
5936             push @$cmd, "zeroinit:$dst_path";
5937         } else {
5938             push @$cmd, $dst_path;
5939         }
5940
5941         my $parser = sub {
5942             my $line = shift;
5943             if($line =~ m/\((\S+)\/100\%\)/){
5944                 my $percent = $1;
5945                 my $transferred = int($size * $percent / 100);
5946                 my $remaining = $size - $transferred;
5947
5948                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5949             }
5950
5951         };
5952
5953         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5954         my $err = $@;
5955         die "copy failed: $err" if $err;
5956     }
5957 }
5958
5959 sub qemu_img_format {
5960     my ($scfg, $volname) = @_;
5961
5962     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5963         return $1;
5964     } else {
5965         return "raw";
5966     }
5967 }
5968
5969 sub qemu_drive_mirror {
5970     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
5971
5972     $jobs = {} if !$jobs;
5973
5974     my $qemu_target;
5975     my $format;
5976     $jobs->{"drive-$drive"} = {};
5977
5978     if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
5979         my $server = $1;
5980         my $port = $2;
5981         my $exportname = $3;
5982
5983         $format = "nbd";
5984         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
5985         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
5986         my $cmd = ['socat', '-T30', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=5"];
5987
5988         my $pid = fork();
5989         if (!defined($pid)) {
5990             die "forking socat tunnel failed\n";
5991         } elsif ($pid == 0) {
5992             exec(@$cmd);
5993             warn "exec failed: $!\n";
5994             POSIX::_exit(-1);
5995         }
5996         $jobs->{"drive-$drive"}->{pid} = $pid;
5997
5998         my $timeout = 0;
5999         while (!-S $unixsocket) {
6000             die "nbd connection helper timed out\n"
6001                 if $timeout++ > 5;
6002             sleep 1;
6003         }
6004     } else {
6005         my $storecfg = PVE::Storage::config();
6006         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6007
6008         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6009
6010         $format = qemu_img_format($dst_scfg, $dst_volname);
6011
6012         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6013
6014         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6015     }
6016
6017     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6018     $opts->{format} = $format if $format;
6019
6020     print "drive mirror is starting for drive-$drive\n";
6021
6022     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6023
6024     if (my $err = $@) {
6025         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6026         die "mirroring error: $err";
6027     }
6028
6029     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6030 }
6031
6032 sub qemu_drive_mirror_monitor {
6033     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6034
6035     eval {
6036         my $err_complete = 0;
6037
6038         while (1) {
6039             die "storage migration timed out\n" if $err_complete > 300;
6040
6041             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6042
6043             my $running_mirror_jobs = {};
6044             foreach my $stat (@$stats) {
6045                 next if $stat->{type} ne 'mirror';
6046                 $running_mirror_jobs->{$stat->{device}} = $stat;
6047             }
6048
6049             my $readycounter = 0;
6050
6051             foreach my $job (keys %$jobs) {
6052
6053                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6054                     print "$job : finished\n";
6055                     delete $jobs->{$job};
6056                     next;
6057                 }
6058
6059                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6060
6061                 my $busy = $running_mirror_jobs->{$job}->{busy};
6062                 my $ready = $running_mirror_jobs->{$job}->{ready};
6063                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6064                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6065                     my $remaining = $total - $transferred;
6066                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6067
6068                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6069                 }
6070
6071                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6072             }
6073
6074             last if scalar(keys %$jobs) == 0;
6075
6076             if ($readycounter == scalar(keys %$jobs)) {
6077                 print "all mirroring jobs are ready \n";
6078                 last if $skipcomplete; #do the complete later
6079
6080                 if ($vmiddst && $vmiddst != $vmid) {
6081                     if ($qga) {
6082                         print "freeze filesystem\n";
6083                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6084                     } else {
6085                         print "suspend vm\n";
6086                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6087                     }
6088
6089                     # if we clone a disk for a new target vm, we don't switch the disk
6090                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6091
6092                     if ($qga) {
6093                         print "unfreeze filesystem\n";
6094                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6095                     } else {
6096                         print "resume vm\n";
6097                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6098                     }
6099
6100                     last;
6101                 } else {
6102
6103                     foreach my $job (keys %$jobs) {
6104                         # try to switch the disk if source and destination are on the same guest
6105                         print "$job: Completing block job...\n";
6106
6107                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6108                         if ($@ =~ m/cannot be completed/) {
6109                             print "$job: Block job cannot be completed, try again.\n";
6110                             $err_complete++;
6111                         }else {
6112                             print "$job: Completed successfully.\n";
6113                             $jobs->{$job}->{complete} = 1;
6114                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6115                         }
6116                     }
6117                 }
6118             }
6119             sleep 1;
6120         }
6121     };
6122     my $err = $@;
6123
6124     if ($err) {
6125         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6126         die "mirroring error: $err";
6127     }
6128
6129 }
6130
6131 sub qemu_blockjobs_cancel {
6132     my ($vmid, $jobs) = @_;
6133
6134     foreach my $job (keys %$jobs) {
6135         print "$job: Cancelling block job\n";
6136         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6137         $jobs->{$job}->{cancel} = 1;
6138     }
6139
6140     while (1) {
6141         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6142
6143         my $running_jobs = {};
6144         foreach my $stat (@$stats) {
6145             $running_jobs->{$stat->{device}} = $stat;
6146         }
6147
6148         foreach my $job (keys %$jobs) {
6149
6150             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6151                 print "$job: Done.\n";
6152                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6153                 delete $jobs->{$job};
6154             }
6155         }
6156
6157         last if scalar(keys %$jobs) == 0;
6158
6159         sleep 1;
6160     }
6161 }
6162
6163 sub qemu_blockjobs_finish_tunnel {
6164    my ($vmid, $job, $cpid) = @_;
6165
6166    return if !$cpid;
6167
6168    for (my $i = 1; $i < 20; $i++) {
6169         my $waitpid = waitpid($cpid, WNOHANG);
6170         last if (defined($waitpid) && ($waitpid == $cpid));
6171
6172         if ($i == 10) {
6173             kill(15, $cpid);
6174         } elsif ($i >= 15) {
6175             kill(9, $cpid);
6176         }
6177         sleep (1);
6178     }
6179     unlink "/run/qemu-server/$vmid.mirror-$job";
6180 }
6181
6182 sub clone_disk {
6183     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6184         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6185
6186     my $newvolid;
6187
6188     if (!$full) {
6189         print "create linked clone of drive $drivename ($drive->{file})\n";
6190         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6191         push @$newvollist, $newvolid;
6192     } else {
6193
6194         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6195         $storeid = $storage if $storage;
6196
6197         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6198         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6199
6200         print "create full clone of drive $drivename ($drive->{file})\n";
6201         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, undef, ($size/1024));
6202         push @$newvollist, $newvolid;
6203
6204         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6205
6206         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6207         if (!$running || $snapname) {
6208             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6209         } else {
6210
6211             my $kvmver = get_running_qemu_version ($vmid);
6212             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6213                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6214                     if $drive->{iothread};
6215             }
6216
6217             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6218         }
6219     }
6220
6221     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6222
6223     my $disk = $drive;
6224     $disk->{format} = undef;
6225     $disk->{file} = $newvolid;
6226     $disk->{size} = $size;
6227
6228     return $disk;
6229 }
6230
6231 # this only works if VM is running
6232 sub get_current_qemu_machine {
6233     my ($vmid) = @_;
6234
6235     my $cmd = { execute => 'query-machines', arguments => {} };
6236     my $res = vm_qmp_command($vmid, $cmd);
6237
6238     my ($current, $default);
6239     foreach my $e (@$res) {
6240         $default = $e->{name} if $e->{'is-default'};
6241         $current = $e->{name} if $e->{'is-current'};
6242     }
6243
6244     # fallback to the default machine if current is not supported by qemu
6245     return $current || $default || 'pc';
6246 }
6247
6248 sub get_running_qemu_version {
6249     my ($vmid) = @_;
6250     my $cmd = { execute => 'query-version', arguments => {} };
6251     my $res = vm_qmp_command($vmid, $cmd);
6252     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6253 }
6254
6255 sub qemu_machine_feature_enabled {
6256     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6257
6258     my $current_major;
6259     my $current_minor;
6260
6261     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6262
6263         $current_major = $3;
6264         $current_minor = $4;
6265
6266     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6267
6268         $current_major = $1;
6269         $current_minor = $2;
6270     }
6271
6272     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6273
6274
6275 }
6276
6277 sub qemu_machine_pxe {
6278     my ($vmid, $conf, $machine) = @_;
6279
6280     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6281
6282     foreach my $opt (keys %$conf) {
6283         next if $opt !~ m/^net(\d+)$/;
6284         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6285         next if !$net;
6286         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6287         return $machine.".pxe" if $romfile =~ m/pxe/;
6288         last;
6289     }
6290
6291     return $machine;
6292 }
6293
6294 sub qemu_use_old_bios_files {
6295     my ($machine_type) = @_;
6296
6297     return if !$machine_type;
6298
6299     my $use_old_bios_files = undef;
6300
6301     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6302         $machine_type = $1;
6303         $use_old_bios_files = 1;
6304     } else {
6305         my $kvmver = kvm_user_version();
6306         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6307         # load new efi bios files on migration. So this hack is required to allow
6308         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6309         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6310         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6311     }
6312
6313     return ($use_old_bios_files, $machine_type);
6314 }
6315
6316 sub create_efidisk {
6317     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6318
6319     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6320
6321     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6322     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6323     PVE::Storage::activate_volumes($storecfg, [$volid]);
6324
6325     my $path = PVE::Storage::path($storecfg, $volid);
6326     eval {
6327         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6328     };
6329     die "Copying EFI vars image failed: $@" if $@;
6330
6331     return ($volid, $vars_size);
6332 }
6333
6334 sub lspci {
6335
6336     my $devices = {};
6337
6338     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6339             my (undef, $id, $function) = @_;
6340             my $res = { id => $id, function => $function};
6341             push @{$devices->{$id}}, $res;
6342     });
6343
6344     # Entries should be sorted by functions.
6345     foreach my $id (keys %$devices) {
6346         my $dev = $devices->{$id};
6347         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6348     }
6349
6350     return $devices;
6351 }
6352
6353 sub vm_iothreads_list {
6354     my ($vmid) = @_;
6355
6356     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6357
6358     my $iothreads = {};
6359     foreach my $iothread (@$res) {
6360         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6361     }
6362
6363     return $iothreads;
6364 }
6365
6366 sub scsihw_infos {
6367     my ($conf, $drive) = @_;
6368
6369     my $maxdev = 0;
6370
6371     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6372         $maxdev = 7;
6373     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6374         $maxdev = 1;
6375     } else {
6376         $maxdev = 256;
6377     }
6378
6379     my $controller = int($drive->{index} / $maxdev);
6380     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6381
6382     return ($maxdev, $controller, $controller_prefix);
6383 }
6384
6385 sub add_hyperv_enlightenments {
6386     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6387
6388     return if $winversion < 6;
6389     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6390
6391     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6392
6393     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6394         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6395         push @$cpuFlags , 'hv_vapic';
6396         push @$cpuFlags , 'hv_time';
6397     } else {
6398         push @$cpuFlags , 'hv_spinlocks=0xffff';
6399     }
6400
6401     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6402         push @$cpuFlags , 'hv_reset';
6403         push @$cpuFlags , 'hv_vpindex';
6404         push @$cpuFlags , 'hv_runtime';
6405     }
6406
6407     if ($winversion >= 7) {
6408         push @$cpuFlags , 'hv_relaxed';
6409     }
6410 }
6411
6412 sub windows_version {
6413     my ($ostype) = @_;
6414
6415     return 0 if !$ostype;
6416
6417     my $winversion = 0;
6418
6419     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6420         $winversion = 5;
6421     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6422         $winversion = 6;
6423     } elsif ($ostype =~ m/^win(\d+)$/) {
6424         $winversion = $1;
6425     }
6426
6427     return $winversion;
6428 }
6429
6430 sub resolve_dst_disk_format {
6431         my ($storecfg, $storeid, $src_volname, $format) = @_;
6432         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6433
6434         if (!$format) {
6435             # if no target format is specified, use the source disk format as hint
6436             if ($src_volname) {
6437                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6438                 $format = qemu_img_format($scfg, $src_volname);
6439             } else {
6440                 return $defFormat;
6441             }
6442         }
6443
6444         # test if requested format is supported - else use default
6445         my $supported = grep { $_ eq $format } @$validFormats;
6446         $format = $defFormat if !$supported;
6447         return $format;
6448 }
6449
6450 sub resolve_first_disk {
6451     my $conf = shift;
6452     my @disks = PVE::QemuServer::valid_drive_names();
6453     my $firstdisk;
6454     foreach my $ds (reverse @disks) {
6455         next if !$conf->{$ds};
6456         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6457         next if PVE::QemuServer::drive_is_cdrom($disk);
6458         $firstdisk = $ds;
6459     }
6460     return $firstdisk;
6461 }
6462
6463 sub generate_smbios1_uuid {
6464     my ($uuid, $uuid_str);
6465     UUID::generate($uuid);
6466     UUID::unparse($uuid, $uuid_str);
6467     return "uuid=$uuid_str";
6468 }
6469
6470 # bash completion helper
6471
6472 sub complete_backup_archives {
6473     my ($cmdname, $pname, $cvalue) = @_;
6474
6475     my $cfg = PVE::Storage::config();
6476
6477     my $storeid;
6478
6479     if ($cvalue =~ m/^([^:]+):/) {
6480         $storeid = $1;
6481     }
6482
6483     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6484
6485     my $res = [];
6486     foreach my $id (keys %$data) {
6487         foreach my $item (@{$data->{$id}}) {
6488             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6489             push @$res, $item->{volid} if defined($item->{volid});
6490         }
6491     }
6492
6493     return $res;
6494 }
6495
6496 my $complete_vmid_full = sub {
6497     my ($running) = @_;
6498
6499     my $idlist = vmstatus();
6500
6501     my $res = [];
6502
6503     foreach my $id (keys %$idlist) {
6504         my $d = $idlist->{$id};
6505         if (defined($running)) {
6506             next if $d->{template};
6507             next if $running && $d->{status} ne 'running';
6508             next if !$running && $d->{status} eq 'running';
6509         }
6510         push @$res, $id;
6511
6512     }
6513     return $res;
6514 };
6515
6516 sub complete_vmid {
6517     return &$complete_vmid_full();
6518 }
6519
6520 sub complete_vmid_stopped {
6521     return &$complete_vmid_full(0);
6522 }
6523
6524 sub complete_vmid_running {
6525     return &$complete_vmid_full(1);
6526 }
6527
6528 sub complete_storage {
6529
6530     my $cfg = PVE::Storage::config();
6531     my $ids = $cfg->{ids};
6532
6533     my $res = [];
6534     foreach my $sid (keys %$ids) {
6535         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6536         next if !$ids->{$sid}->{content}->{images};
6537         push @$res, $sid;
6538     }
6539
6540     return $res;
6541 }
6542
6543 sub nbd_stop {
6544     my ($vmid) = @_;
6545
6546     vm_mon_cmd($vmid, 'nbd-server-stop');
6547 }
6548
6549 1;