PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42 my $OVMF_IMG = '/usr/share/kvm/OVMF-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 # Note about locking: we use flock on the config file protect
  49 # against concurent actions.
  50 # Aditionaly, we have a 'lock' setting in the config file. This
  51 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  52 # allowed when such lock is set. But you can ignore this kind of
  53 # lock with the --skiplock flag.
  54
  55 cfs_register_file('/qemu-server/',
  56                   \&parse_vm_config,
  57                   \&write_vm_config);
  58
  59 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  60     description => "Some command save/restore state from this location.",
  61     type => 'string',
  62     maxLength => 128,
  63     optional => 1,
  64 });
  65
  66 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  67     description => "The name of the snapshot.",
  68     type => 'string', format => 'pve-configid',
  69     maxLength => 40,
  70 });
  71
  72 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  73     type => 'string',
  74     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  75     description => "The drive's backing file's data format.",
  76     optional => 1,
  77 });
  78
  79 #no warnings 'redefine';
  80
  81 sub cgroups_write {
  82    my ($controller, $vmid, $option, $value) = @_;
  83
  84    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  85    PVE::ProcFSTools::write_proc_entry($path, $value);
  86
  87 }
  88
  89 my $nodename = PVE::INotify::nodename();
  90
  91 mkdir "/etc/pve/nodes/$nodename";
  92 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  93 mkdir $confdir;
  94
  95 my $var_run_tmpdir = "/var/run/qemu-server";
  96 mkdir $var_run_tmpdir;
  97
  98 my $lock_dir = "/var/lock/qemu-server";
  99 mkdir $lock_dir;
 100
 101 my $pcisysfs = "/sys/bus/pci";
 102
 103 my $cpu_vendor_list = {
 104     # Intel CPUs
 105     486 => 'GenuineIntel',
 106     pentium => 'GenuineIntel',
 107     pentium2  => 'GenuineIntel',
 108     pentium3  => 'GenuineIntel',
 109     coreduo => 'GenuineIntel',
 110     core2duo => 'GenuineIntel',
 111     Conroe  => 'GenuineIntel',
 112     Penryn  => 'GenuineIntel',
 113     Nehalem  => 'GenuineIntel',
 114     Westmere => 'GenuineIntel',
 115     SandyBridge => 'GenuineIntel',
 116     IvyBridge => 'GenuineIntel',
 117     Haswell => 'GenuineIntel',
 118     'Haswell-noTSX' => 'GenuineIntel',
 119     Broadwell => 'GenuineIntel',
 120     'Broadwell-noTSX' => 'GenuineIntel',
 121     'Skylake-Client' => 'GenuineIntel',
 122
 123     # AMD CPUs
 124     athlon => 'AuthenticAMD',
 125     phenom  => 'AuthenticAMD',
 126     Opteron_G1  => 'AuthenticAMD',
 127     Opteron_G2  => 'AuthenticAMD',
 128     Opteron_G3  => 'AuthenticAMD',
 129     Opteron_G4  => 'AuthenticAMD',
 130     Opteron_G5  => 'AuthenticAMD',
 131
 132     # generic types, use vendor from host node
 133     host => 'default',
 134     kvm32 => 'default',
 135     kvm64 => 'default',
 136     qemu32 => 'default',
 137     qemu64 => 'default',
 138 };
 139
 140 my $cpu_fmt = {
 141     cputype => {
 142         description => "Emulated CPU type.",
 143         type => 'string',
 144         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 145         default => 'kvm64',
 146         default_key => 1,
 147     },
 148     hidden => {
 149         description => "Do not identify as a KVM virtual machine.",
 150         type => 'boolean',
 151         optional => 1,
 152         default => 0
 153     },
 154 };
 155
 156 my $watchdog_fmt = {
 157     model => {
 158         default_key => 1,
 159         type => 'string',
 160         enum => [qw(i6300esb ib700)],
 161         description => "Watchdog type to emulate.",
 162         default => 'i6300esb',
 163         optional => 1,
 164     },
 165     action => {
 166         type => 'string',
 167         enum => [qw(reset shutdown poweroff pause debug none)],
 168         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 169         optional => 1,
 170     },
 171 };
 172 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 173
 174 my $confdesc = {
 175     onboot => {
 176         optional => 1,
 177         type => 'boolean',
 178         description => "Specifies whether a VM will be started during system bootup.",
 179         default => 0,
 180     },
 181     autostart => {
 182         optional => 1,
 183         type => 'boolean',
 184         description => "Automatic restart after crash (currently ignored).",
 185         default => 0,
 186     },
 187     hotplug => {
 188         optional => 1,
 189         type => 'string', format => 'pve-hotplug-features',
 190         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 191         default => 'network,disk,usb',
 192     },
 193     reboot => {
 194         optional => 1,
 195         type => 'boolean',
 196         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 197         default => 1,
 198     },
 199     lock => {
 200         optional => 1,
 201         type => 'string',
 202         description => "Lock/unlock the VM.",
 203         enum => [qw(migrate backup snapshot rollback)],
 204     },
 205     cpulimit => {
 206         optional => 1,
 207         type => 'number',
 208         description => "Limit of CPU usage.",
 209         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 210         minimum => 0,
 211         maximum => 128,
 212         default => 0,
 213     },
 214     cpuunits => {
 215         optional => 1,
 216         type => 'integer',
 217         description => "CPU weight for a VM.",
 218         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 219         minimum => 0,
 220         maximum => 500000,
 221         default => 1024,
 222     },
 223     memory => {
 224         optional => 1,
 225         type => 'integer',
 226         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 227         minimum => 16,
 228         default => 512,
 229     },
 230     balloon => {
 231         optional => 1,
 232         type => 'integer',
 233         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 234         minimum => 0,
 235     },
 236     shares => {
 237         optional => 1,
 238         type => 'integer',
 239         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 240         minimum => 0,
 241         maximum => 50000,
 242         default => 1000,
 243     },
 244     keyboard => {
 245         optional => 1,
 246         type => 'string',
 247         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 248         enum => PVE::Tools::kvmkeymaplist(),
 249         default => 'en-us',
 250     },
 251     name => {
 252         optional => 1,
 253         type => 'string', format => 'dns-name',
 254         description => "Set a name for the VM. Only used on the configuration web interface.",
 255     },
 256     scsihw => {
 257         optional => 1,
 258         type => 'string',
 259         description => "SCSI controller model",
 260         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 261         default => 'lsi',
 262     },
 263     description => {
 264         optional => 1,
 265         type => 'string',
 266         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 267     },
 268     ostype => {
 269         optional => 1,
 270         type => 'string',
 271         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 272         description => "Specify guest operating system.",
 273         verbose_description => <<EODESC,
 274 Specify guest operating system. This is used to enable special
 275 optimization/features for specific operating systems:
 276
 277 [horizontal]
 278 other;; unspecified OS
 279 wxp;; Microsoft Windows XP
 280 w2k;; Microsoft Windows 2000
 281 w2k3;; Microsoft Windows 2003
 282 w2k8;; Microsoft Windows 2008
 283 wvista;; Microsoft Windows Vista
 284 win7;; Microsoft Windows 7
 285 win8;; Microsoft Windows 8/2012
 286 l24;; Linux 2.4 Kernel
 287 l26;; Linux 2.6/3.X Kernel
 288 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 289 EODESC
 290     },
 291     boot => {
 292         optional => 1,
 293         type => 'string',
 294         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 295         pattern => '[acdn]{1,4}',
 296         default => 'cdn',
 297     },
 298     bootdisk => {
 299         optional => 1,
 300         type => 'string', format => 'pve-qm-bootdisk',
 301         description => "Enable booting from specified disk.",
 302         pattern => '(ide|sata|scsi|virtio)\d+',
 303     },
 304     smp => {
 305         optional => 1,
 306         type => 'integer',
 307         description => "The number of CPUs. Please use option -sockets instead.",
 308         minimum => 1,
 309         default => 1,
 310     },
 311     sockets => {
 312         optional => 1,
 313         type => 'integer',
 314         description => "The number of CPU sockets.",
 315         minimum => 1,
 316         default => 1,
 317     },
 318     cores => {
 319         optional => 1,
 320         type => 'integer',
 321         description => "The number of cores per socket.",
 322         minimum => 1,
 323         default => 1,
 324     },
 325     numa => {
 326         optional => 1,
 327         type => 'boolean',
 328         description => "Enable/disable NUMA.",
 329         default => 0,
 330     },
 331     hugepages => {
 332         optional => 1,
 333         type => 'string',
 334         description => "Enable/disable hugepages memory.",
 335         enum => [qw(any 2 1024)],
 336     },
 337     vcpus => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "Number of hotplugged vcpus.",
 341         minimum => 1,
 342         default => 0,
 343     },
 344     acpi => {
 345         optional => 1,
 346         type => 'boolean',
 347         description => "Enable/disable ACPI.",
 348         default => 1,
 349     },
 350     agent => {
 351         optional => 1,
 352         type => 'boolean',
 353         description => "Enable/disable Qemu GuestAgent.",
 354         default => 0,
 355     },
 356     kvm => {
 357         optional => 1,
 358         type => 'boolean',
 359         description => "Enable/disable KVM hardware virtualization.",
 360         default => 1,
 361     },
 362     tdf => {
 363         optional => 1,
 364         type => 'boolean',
 365         description => "Enable/disable time drift fix.",
 366         default => 0,
 367     },
 368     localtime => {
 369         optional => 1,
 370         type => 'boolean',
 371         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 372     },
 373     freeze => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 377     },
 378     vga => {
 379         optional => 1,
 380         type => 'string',
 381         description => "Select the VGA type.",
 382         verbose_description => "Select the VGA type. If you want to use high resolution" .
 383             " modes (>= 1280x1024x16) then you should use the options " .
 384             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 385             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 386             "display sever. For win* OS you can select how many independent " .
 387             "displays you want, Linux guests can add displays them self. " .
 388             "You can also run without any graphic card, using a serial device" .
 389             " as terminal.",
 390         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 391     },
 392     watchdog => {
 393         optional => 1,
 394         type => 'string', format => 'pve-qm-watchdog',
 395         description => "Create a virtual hardware watchdog device.",
 396         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 397             " (by a guest action), the watchdog must be periodically polled " .
 398             "by an agent inside the guest or else the watchdog will reset " .
 399             "the guest (or execute the respective action specified)",
 400     },
 401     startdate => {
 402         optional => 1,
 403         type => 'string',
 404         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 405         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 406         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 407         default => 'now',
 408     },
 409     startup =>  get_standard_option('pve-startup-order'),
 410     template => {
 411         optional => 1,
 412         type => 'boolean',
 413         description => "Enable/disable Template.",
 414         default => 0,
 415     },
 416     args => {
 417         optional => 1,
 418         type => 'string',
 419         description => "Arbitrary arguments passed to kvm.",
 420         verbose_description => <<EODESCR,
 421 Arbitrary arguments passed to kvm, for example:
 422
 423 args: -no-reboot -no-hpet
 424
 425 NOTE: this option is for experts only.
 426 EODESCR
 427     },
 428     tablet => {
 429         optional => 1,
 430         type => 'boolean',
 431         default => 1,
 432         description => "Enable/disable the USB tablet device.",
 433         verbose_description => "Enable/disable the USB tablet device. This device is " .
 434             "usually needed to allow absolute mouse positioning with VNC. " .
 435             "Else the mouse runs out of sync with normal VNC clients. " .
 436             "If you're running lots of console-only guests on one host, " .
 437             "you may consider disabling this to save some context switches. " .
 438             "This is turned off by default if you use spice (-vga=qxl).",
 439     },
 440     migrate_speed => {
 441         optional => 1,
 442         type => 'integer',
 443         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 444         minimum => 0,
 445         default => 0,
 446     },
 447     migrate_downtime => {
 448         optional => 1,
 449         type => 'number',
 450         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 451         minimum => 0,
 452         default => 0.1,
 453     },
 454     cdrom => {
 455         optional => 1,
 456         type => 'string', format => 'pve-qm-ide',
 457         typetext => '<volume>',
 458         description => "This is an alias for option -ide2",
 459     },
 460     cpu => {
 461         optional => 1,
 462         description => "Emulated CPU type.",
 463         type => 'string',
 464         format => $cpu_fmt,
 465     },
 466     parent => get_standard_option('pve-snapshot-name', {
 467         optional => 1,
 468         description => "Parent snapshot name. This is used internally, and should not be modified.",
 469     }),
 470     snaptime => {
 471         optional => 1,
 472         description => "Timestamp for snapshots.",
 473         type => 'integer',
 474         minimum => 0,
 475     },
 476     vmstate => {
 477         optional => 1,
 478         type => 'string', format => 'pve-volume-id',
 479         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 480     },
 481     machine => {
 482         description => "Specific the Qemu machine type.",
 483         type => 'string',
 484         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 485         maxLength => 40,
 486         optional => 1,
 487     },
 488     smbios1 => {
 489         description => "Specify SMBIOS type 1 fields.",
 490         type => 'string', format => 'pve-qm-smbios1',
 491         maxLength => 256,
 492         optional => 1,
 493     },
 494     protection => {
 495         optional => 1,
 496         type => 'boolean',
 497         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 498         default => 0,
 499     },
 500     bios => {
 501         optional => 1,
 502         type => 'string',
 503         enum => [ qw(seabios ovmf) ],
 504         description => "Select BIOS implementation.",
 505         default => 'seabios',
 506     },
 507 };
 508
 509 # what about other qemu settings ?
 510 #cpu => 'string',
 511 #machine => 'string',
 512 #fda => 'file',
 513 #fdb => 'file',
 514 #mtdblock => 'file',
 515 #sd => 'file',
 516 #pflash => 'file',
 517 #snapshot => 'bool',
 518 #bootp => 'file',
 519 ##tftp => 'dir',
 520 ##smb => 'dir',
 521 #kernel => 'file',
 522 #append => 'string',
 523 #initrd => 'file',
 524 ##soundhw => 'string',
 525
 526 while (my ($k, $v) = each %$confdesc) {
 527     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 528 }
 529
 530 my $MAX_IDE_DISKS = 4;
 531 my $MAX_SCSI_DISKS = 14;
 532 my $MAX_VIRTIO_DISKS = 16;
 533 my $MAX_SATA_DISKS = 6;
 534 my $MAX_USB_DEVICES = 5;
 535 my $MAX_NETS = 32;
 536 my $MAX_UNUSED_DISKS = 8;
 537 my $MAX_HOSTPCI_DEVICES = 4;
 538 my $MAX_SERIAL_PORTS = 4;
 539 my $MAX_PARALLEL_PORTS = 3;
 540 my $MAX_NUMA = 8;
 541
 542 my $numa_fmt = {
 543     cpus => {
 544         type => "string",
 545         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 546         description => "CPUs accessing this NUMA node.",
 547         format_description => "id[-id];...",
 548     },
 549     memory => {
 550         type => "number",
 551         description => "Amount of memory this NUMA node provides.",
 552         optional => 1,
 553     },
 554     hostnodes => {
 555         type => "string",
 556         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 557         description => "Host NUMA nodes to use.",
 558         format_description => "id[-id];...",
 559         optional => 1,
 560     },
 561     policy => {
 562         type => 'string',
 563         enum => [qw(preferred bind interleave)],
 564         description => "NUMA allocation policy.",
 565         optional => 1,
 566     },
 567 };
 568 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 569 my $numadesc = {
 570     optional => 1,
 571     type => 'string', format => $numa_fmt,
 572     description => "NUMA topology.",
 573 };
 574 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 575
 576 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 577     $confdesc->{"numa$i"} = $numadesc;
 578 }
 579
 580 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 581                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 582                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 583 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 584
 585 my $net_fmt_bridge_descr = <<__EOD__;
 586 Bridge to attach the network device to. The Proxmox VE standard bridge
 587 is called 'vmbr0'.
 588
 589 If you do not specify a bridge, we create a kvm user (NATed) network
 590 device, which provides DHCP and DNS services. The following addresses
 591 are used:
 592
 593  10.0.2.2   Gateway
 594  10.0.2.3   DNS Server
 595  10.0.2.4   SMB Server
 596
 597 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 598 __EOD__
 599
 600 my $net_fmt = {
 601     macaddr => {
 602         type => 'string',
 603         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 604         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 605         format_description => "XX:XX:XX:XX:XX:XX",
 606         optional => 1,
 607     },
 608     model => {
 609         type => 'string',
 610         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 611         enum => $nic_model_list,
 612         default_key => 1,
 613     },
 614     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 615     bridge => {
 616         type => 'string',
 617         description => $net_fmt_bridge_descr,
 618         format_description => 'bridge',
 619         optional => 1,
 620     },
 621     queues => {
 622         type => 'integer',
 623         minimum => 0, maximum => 16,
 624         description => 'Number of packet queues to be used on the device.',
 625         optional => 1,
 626     },
 627     rate => {
 628         type => 'number',
 629         minimum => 0,
 630         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 631         optional => 1,
 632     },
 633     tag => {
 634         type => 'integer',
 635         minimum => 1, maximum => 4094,
 636         description => 'VLAN tag to apply to packets on this interface.',
 637         optional => 1,
 638     },
 639     trunks => {
 640         type => 'string',
 641         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 642         description => 'VLAN trunks to pass through this interface.',
 643         format_description => 'vlanid[;vlanid...]',
 644         optional => 1,
 645     },
 646     firewall => {
 647         type => 'boolean',
 648         description => 'Whether this interface should be protected by the firewall.',
 649         optional => 1,
 650     },
 651     link_down => {
 652         type => 'boolean',
 653         description => 'Whether this interface should be disconnected (like pulling the plug).',
 654         optional => 1,
 655     },
 656 };
 657
 658 my $netdesc = {
 659     optional => 1,
 660     type => 'string', format => $net_fmt,
 661     description => "Specify network devices.",
 662 };
 663
 664 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 665
 666 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 667     $confdesc->{"net$i"} = $netdesc;
 668 }
 669
 670 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 671 sub verify_volume_id_or_qm_path {
 672     my ($volid, $noerr) = @_;
 673
 674     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 675         return $volid;
 676     }
 677
 678     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 679     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 680     if ($@) {
 681         return undef if $noerr;
 682         die $@;
 683     }
 684     return $volid;
 685 }
 686
 687 my $drivename_hash;
 688
 689 my %drivedesc_base = (
 690     volume => { alias => 'file' },
 691     file => {
 692         type => 'string',
 693         format => 'pve-volume-id-or-qm-path',
 694         default_key => 1,
 695         format_description => 'volume',
 696         description => "The drive's backing volume.",
 697     },
 698     media => {
 699         type => 'string',
 700         enum => [qw(cdrom disk)],
 701         description => "The drive's media type.",
 702         default => 'disk',
 703         optional => 1
 704     },
 705     cyls => {
 706         type => 'integer',
 707         description => "Force the drive's physical geometry to have a specific cylinder count.",
 708         optional => 1
 709     },
 710     heads => {
 711         type => 'integer',
 712         description => "Force the drive's physical geometry to have a specific head count.",
 713         optional => 1
 714     },
 715     secs => {
 716         type => 'integer',
 717         description => "Force the drive's physical geometry to have a specific sector count.",
 718         optional => 1
 719     },
 720     trans => {
 721         type => 'string',
 722         enum => [qw(none lba auto)],
 723         description => "Force disk geometry bios translation mode.",
 724         optional => 1,
 725     },
 726     snapshot => {
 727         type => 'boolean',
 728         description => "Whether the drive should be included when making snapshots.",
 729         optional => 1,
 730     },
 731     cache => {
 732         type => 'string',
 733         enum => [qw(none writethrough writeback unsafe directsync)],
 734         description => "The drive's cache mode",
 735         optional => 1,
 736     },
 737     format => get_standard_option('pve-qm-image-format'),
 738     size => {
 739         type => 'string',
 740         format => 'disk-size',
 741         format_description => 'DiskSize',
 742         description => "Disk size. This is purely informational and has no effect.",
 743         optional => 1,
 744     },
 745     backup => {
 746         type => 'boolean',
 747         description => "Whether the drive should be included when making backups.",
 748         optional => 1,
 749     },
 750     replicate => {
 751         type => 'boolean',
 752         description => 'Whether the drive should considered for replication jobs.',
 753         optional => 1,
 754         default => 1,
 755     },
 756     rerror => {
 757         type => 'string',
 758         enum => [qw(ignore report stop)],
 759         description => 'Read error action.',
 760         optional => 1,
 761     },
 762     werror => {
 763         type => 'string',
 764         enum => [qw(enospc ignore report stop)],
 765         description => 'Write error action.',
 766         optional => 1,
 767     },
 768     aio => {
 769         type => 'string',
 770         enum => [qw(native threads)],
 771         description => 'AIO type to use.',
 772         optional => 1,
 773     },
 774     discard => {
 775         type => 'string',
 776         enum => [qw(ignore on)],
 777         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 778         optional => 1,
 779     },
 780     detect_zeroes => {
 781         type => 'boolean',
 782         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 783         optional => 1,
 784     },
 785     serial => {
 786         type => 'string',
 787         format => 'urlencoded',
 788         format_description => 'serial',
 789         maxLength => 20*3, # *3 since it's %xx url enoded
 790         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 791         optional => 1,
 792     }
 793 );
 794
 795 my %iothread_fmt = ( iothread => {
 796         type => 'boolean',
 797         description => "Whether to use iothreads for this drive",
 798         optional => 1,
 799 });
 800
 801 my %model_fmt = (
 802     model => {
 803         type => 'string',
 804         format => 'urlencoded',
 805         format_description => 'model',
 806         maxLength => 40*3, # *3 since it's %xx url enoded
 807         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 808         optional => 1,
 809     },
 810 );
 811
 812 my %queues_fmt = (
 813     queues => {
 814         type => 'integer',
 815         description => "Number of queues.",
 816         minimum => 2,
 817         optional => 1
 818     }
 819 );
 820
 821 my %scsiblock_fmt = (
 822     scsiblock => {
 823         type => 'boolean',
 824         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 825         optional => 1,
 826         default => 0,
 827     },
 828 );
 829
 830 my $add_throttle_desc = sub {
 831     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 832     my $d = {
 833         type => $type,
 834         format_description => $unit,
 835         description => "Maximum $what in $longunit.",
 836         optional => 1,
 837     };
 838     $d->{minimum} = $minimum if defined($minimum);
 839     $drivedesc_base{$key} = $d;
 840 };
 841 # throughput: (leaky bucket)
 842 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 843 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 844 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 845 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 846 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 847 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 848 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 849 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 850 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 851
 852 # pools: (pool of IO before throttling starts taking effect)
 853 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 854 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 855 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 856 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 857 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 858 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 859
 860 # burst lengths
 861 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 862 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 863 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 864 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 865 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 866 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 867
 868 # legacy support
 869 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
 870 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
 871 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
 872 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
 873
 874 my $ide_fmt = {
 875     %drivedesc_base,
 876     %model_fmt,
 877 };
 878 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 879
 880 my $idedesc = {
 881     optional => 1,
 882     type => 'string', format => $ide_fmt,
 883     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 884 };
 885 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 886
 887 my $scsi_fmt = {
 888     %drivedesc_base,
 889     %iothread_fmt,
 890     %queues_fmt,
 891     %scsiblock_fmt,
 892 };
 893 my $scsidesc = {
 894     optional => 1,
 895     type => 'string', format => $scsi_fmt,
 896     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 897 };
 898 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 899
 900 my $sata_fmt = {
 901     %drivedesc_base,
 902 };
 903 my $satadesc = {
 904     optional => 1,
 905     type => 'string', format => $sata_fmt,
 906     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 907 };
 908 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 909
 910 my $virtio_fmt = {
 911     %drivedesc_base,
 912     %iothread_fmt,
 913 };
 914 my $virtiodesc = {
 915     optional => 1,
 916     type => 'string', format => $virtio_fmt,
 917     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 918 };
 919 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 920
 921 my $alldrive_fmt = {
 922     %drivedesc_base,
 923     %iothread_fmt,
 924     %model_fmt,
 925     %queues_fmt,
 926     %scsiblock_fmt,
 927 };
 928
 929 my $efidisk_fmt = {
 930     volume => { alias => 'file' },
 931     file => {
 932         type => 'string',
 933         format => 'pve-volume-id-or-qm-path',
 934         default_key => 1,
 935         format_description => 'volume',
 936         description => "The drive's backing volume.",
 937     },
 938     format => get_standard_option('pve-qm-image-format'),
 939     size => {
 940         type => 'string',
 941         format => 'disk-size',
 942         format_description => 'DiskSize',
 943         description => "Disk size. This is purely informational and has no effect.",
 944         optional => 1,
 945     },
 946 };
 947
 948 my $efidisk_desc = {
 949     optional => 1,
 950     type => 'string', format => $efidisk_fmt,
 951     description => "Configure a Disk for storing EFI vars",
 952 };
 953
 954 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 955
 956 my $usb_fmt = {
 957     host => {
 958         default_key => 1,
 959         type => 'string', format => 'pve-qm-usb-device',
 960         format_description => 'HOSTUSBDEVICE|spice',
 961         description => <<EODESCR,
 962 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 963
 964  'bus-port(.port)*' (decimal numbers) or
 965  'vendor_id:product_id' (hexadeciaml numbers) or
 966  'spice'
 967
 968 You can use the 'lsusb -t' command to list existing usb devices.
 969
 970 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 971
 972 The value 'spice' can be used to add a usb redirection devices for spice.
 973 EODESCR
 974     },
 975     usb3 => {
 976         optional => 1,
 977         type => 'boolean',
 978         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 979         default => 0,
 980     },
 981 };
 982
 983 my $usbdesc = {
 984     optional => 1,
 985     type => 'string', format => $usb_fmt,
 986     description => "Configure an USB device (n is 0 to 4).",
 987 };
 988 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 989
 990 # NOTE: the match-groups of this regex are used in parse_hostpci
 991 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 992 my $hostpci_fmt = {
 993     host => {
 994         default_key => 1,
 995         type => 'string',
 996         pattern => qr/$PCIRE(;$PCIRE)*/,
 997         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 998         description => <<EODESCR,
 999 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1000 of PCI virtual functions of the host. HOSTPCIID syntax is:
1001
1002 'bus:dev.func' (hexadecimal numbers)
1003
1004 You can us the 'lspci' command to list existing PCI devices.
1005 EODESCR
1006     },
1007     rombar => {
1008         type => 'boolean',
1009         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1010         optional => 1,
1011         default => 1,
1012     },
1013     romfile => {
1014         type => 'string',
1015         pattern => '[^,;]+',
1016         format_description => 'string',
1017         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1018         optional => 1,
1019     },
1020     pcie => {
1021         type => 'boolean',
1022         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1023         optional => 1,
1024         default => 0,
1025     },
1026     'x-vga' => {
1027         type => 'boolean',
1028         description =>  "Enable vfio-vga device support.",
1029         optional => 1,
1030         default => 0,
1031     },
1032 };
1033 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1034
1035 my $hostpcidesc = {
1036         optional => 1,
1037         type => 'string', format => 'pve-qm-hostpci',
1038         description => "Map host PCI devices into guest.",
1039         verbose_description =>  <<EODESCR,
1040 Map host PCI devices into guest.
1041
1042 NOTE: This option allows direct access to host hardware. So it is no longer
1043 possible to migrate such machines - use with special care.
1044
1045 CAUTION: Experimental! User reported problems with this option.
1046 EODESCR
1047 };
1048 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1049
1050 my $serialdesc = {
1051         optional => 1,
1052         type => 'string',
1053         pattern => '(/dev/.+|socket)',
1054         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1055         verbose_description =>  <<EODESCR,
1056 Create a serial device inside the VM (n is 0 to 3), and pass through a
1057 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1058 host side (use 'qm terminal' to open a terminal connection).
1059
1060 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1061
1062 CAUTION: Experimental! User reported problems with this option.
1063 EODESCR
1064 };
1065
1066 my $paralleldesc= {
1067         optional => 1,
1068         type => 'string',
1069         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1070         description =>  "Map host parallel devices (n is 0 to 2).",
1071         verbose_description =>  <<EODESCR,
1072 Map host parallel devices (n is 0 to 2).
1073
1074 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1075
1076 CAUTION: Experimental! User reported problems with this option.
1077 EODESCR
1078 };
1079
1080 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1081     $confdesc->{"parallel$i"} = $paralleldesc;
1082 }
1083
1084 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1085     $confdesc->{"serial$i"} = $serialdesc;
1086 }
1087
1088 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1089     $confdesc->{"hostpci$i"} = $hostpcidesc;
1090 }
1091
1092 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1093     $drivename_hash->{"ide$i"} = 1;
1094     $confdesc->{"ide$i"} = $idedesc;
1095 }
1096
1097 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1098     $drivename_hash->{"sata$i"} = 1;
1099     $confdesc->{"sata$i"} = $satadesc;
1100 }
1101
1102 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1103     $drivename_hash->{"scsi$i"} = 1;
1104     $confdesc->{"scsi$i"} = $scsidesc ;
1105 }
1106
1107 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1108     $drivename_hash->{"virtio$i"} = 1;
1109     $confdesc->{"virtio$i"} = $virtiodesc;
1110 }
1111
1112 $drivename_hash->{efidisk0} = 1;
1113 $confdesc->{efidisk0} = $efidisk_desc;
1114
1115 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1116     $confdesc->{"usb$i"} = $usbdesc;
1117 }
1118
1119 my $unuseddesc = {
1120     optional => 1,
1121     type => 'string', format => 'pve-volume-id',
1122     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1123 };
1124
1125 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1126     $confdesc->{"unused$i"} = $unuseddesc;
1127 }
1128
1129 my $kvm_api_version = 0;
1130
1131 sub kvm_version {
1132
1133     return $kvm_api_version if $kvm_api_version;
1134
1135     my $fh = IO::File->new("</dev/kvm") ||
1136         return 0;
1137
1138     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1139         $kvm_api_version = $v;
1140     }
1141
1142     $fh->close();
1143
1144     return  $kvm_api_version;
1145 }
1146
1147 my $kvm_user_version;
1148
1149 sub kvm_user_version {
1150
1151     return $kvm_user_version if $kvm_user_version;
1152
1153     $kvm_user_version = 'unknown';
1154
1155     my $code = sub {
1156         my $line = shift;
1157         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1158             $kvm_user_version = $2;
1159         }
1160     };
1161
1162     eval { run_command("kvm -version", outfunc => $code); };
1163     warn $@ if $@;
1164
1165     return $kvm_user_version;
1166
1167 }
1168
1169 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1170
1171 sub valid_drive_names {
1172     # order is important - used to autoselect boot disk
1173     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1174             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1175             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1176             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1177             'efidisk0');
1178 }
1179
1180 sub is_valid_drivename {
1181     my $dev = shift;
1182
1183     return defined($drivename_hash->{$dev});
1184 }
1185
1186 sub option_exists {
1187     my $key = shift;
1188     return defined($confdesc->{$key});
1189 }
1190
1191 sub nic_models {
1192     return $nic_model_list;
1193 }
1194
1195 sub os_list_description {
1196
1197     return {
1198         other => 'Other',
1199         wxp => 'Windows XP',
1200         w2k => 'Windows 2000',
1201         w2k3 =>, 'Windows 2003',
1202         w2k8 => 'Windows 2008',
1203         wvista => 'Windows Vista',
1204         win7 => 'Windows 7',
1205         win8 => 'Windows 8/2012',
1206         win10 => 'Windows 10/2016',
1207         l24 => 'Linux 2.4',
1208         l26 => 'Linux 2.6',
1209     };
1210 }
1211
1212 my $cdrom_path;
1213
1214 sub get_cdrom_path {
1215
1216     return  $cdrom_path if $cdrom_path;
1217
1218     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1219     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1220     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1221 }
1222
1223 sub get_iso_path {
1224     my ($storecfg, $vmid, $cdrom) = @_;
1225
1226     if ($cdrom eq 'cdrom') {
1227         return get_cdrom_path();
1228     } elsif ($cdrom eq 'none') {
1229         return '';
1230     } elsif ($cdrom =~ m|^/|) {
1231         return $cdrom;
1232     } else {
1233         return PVE::Storage::path($storecfg, $cdrom);
1234     }
1235 }
1236
1237 # try to convert old style file names to volume IDs
1238 sub filename_to_volume_id {
1239     my ($vmid, $file, $media) = @_;
1240
1241     if (!($file eq 'none' || $file eq 'cdrom' ||
1242           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1243
1244         return undef if $file =~ m|/|;
1245
1246         if ($media && $media eq 'cdrom') {
1247             $file = "local:iso/$file";
1248         } else {
1249             $file = "local:$vmid/$file";
1250         }
1251     }
1252
1253     return $file;
1254 }
1255
1256 sub verify_media_type {
1257     my ($opt, $vtype, $media) = @_;
1258
1259     return if !$media;
1260
1261     my $etype;
1262     if ($media eq 'disk') {
1263         $etype = 'images';
1264     } elsif ($media eq 'cdrom') {
1265         $etype = 'iso';
1266     } else {
1267         die "internal error";
1268     }
1269
1270     return if ($vtype eq $etype);
1271
1272     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1273 }
1274
1275 sub cleanup_drive_path {
1276     my ($opt, $storecfg, $drive) = @_;
1277
1278     # try to convert filesystem paths to volume IDs
1279
1280     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1281         ($drive->{file} !~ m|^/dev/.+|) &&
1282         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1283         ($drive->{file} !~ m/^\d+$/)) {
1284         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1285         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1286         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1287         verify_media_type($opt, $vtype, $drive->{media});
1288         $drive->{file} = $volid;
1289     }
1290
1291     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1292 }
1293
1294 sub parse_hotplug_features {
1295     my ($data) = @_;
1296
1297     my $res = {};
1298
1299     return $res if $data eq '0';
1300
1301     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1302
1303     foreach my $feature (PVE::Tools::split_list($data)) {
1304         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1305             $res->{$1} = 1;
1306         } else {
1307             die "invalid hotplug feature '$feature'\n";
1308         }
1309     }
1310     return $res;
1311 }
1312
1313 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1314 sub pve_verify_hotplug_features {
1315     my ($value, $noerr) = @_;
1316
1317     return $value if parse_hotplug_features($value);
1318
1319     return undef if $noerr;
1320
1321     die "unable to parse hotplug option\n";
1322 }
1323
1324 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1325 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1326 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1327 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1328 #        [,iothread=on][,serial=serial][,model=model]
1329
1330 sub parse_drive {
1331     my ($key, $data) = @_;
1332
1333     my ($interface, $index);
1334
1335     if ($key =~ m/^([^\d]+)(\d+)$/) {
1336         $interface = $1;
1337         $index = $2;
1338     } else {
1339         return undef;
1340     }
1341
1342     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1343                                      : $confdesc->{$key}->{format};
1344     if (!$desc) {
1345         warn "invalid drive key: $key\n";
1346         return undef;
1347     }
1348     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1349     return undef if !$res;
1350     $res->{interface} = $interface;
1351     $res->{index} = $index;
1352
1353     my $error = 0;
1354     foreach my $opt (qw(bps bps_rd bps_wr)) {
1355         if (my $bps = defined(delete $res->{$opt})) {
1356             if (defined($res->{"m$opt"})) {
1357                 warn "both $opt and m$opt specified\n";
1358                 ++$error;
1359                 next;
1360             }
1361             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1362         }
1363     }
1364
1365     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1366     for my $requirement (
1367         [mbps_max => 'mbps'],
1368         [mbps_rd_max => 'mbps_rd'],
1369         [mbps_wr_max => 'mbps_wr'],
1370         [miops_max => 'miops'],
1371         [miops_rd_max => 'miops_rd'],
1372         [miops_wr_max => 'miops_wr'],
1373         [bps_max_length => 'mbps_max'],
1374         [bps_rd_max_length => 'mbps_rd_max'],
1375         [bps_wr_max_length => 'mbps_wr_max'],
1376         [iops_max_length => 'iops_max'],
1377         [iops_rd_max_length => 'iops_rd_max'],
1378         [iops_wr_max_length => 'iops_wr_max']) {
1379         my ($option, $requires) = @$requirement;
1380         if ($res->{$option} && !$res->{$requires}) {
1381             warn "$option requires $requires\n";
1382             ++$error;
1383         }
1384     }
1385
1386     return undef if $error;
1387
1388     return undef if $res->{mbps_rd} && $res->{mbps};
1389     return undef if $res->{mbps_wr} && $res->{mbps};
1390     return undef if $res->{iops_rd} && $res->{iops};
1391     return undef if $res->{iops_wr} && $res->{iops};
1392
1393     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1394         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1395         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1396         return undef if $res->{interface} eq 'virtio';
1397     }
1398
1399     if (my $size = $res->{size}) {
1400         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1401     }
1402
1403     return $res;
1404 }
1405
1406 sub print_drive {
1407     my ($vmid, $drive) = @_;
1408     my $data = { %$drive };
1409     delete $data->{$_} for qw(index interface);
1410     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1411 }
1412
1413 sub scsi_inquiry {
1414     my($fh, $noerr) = @_;
1415
1416     my $SG_IO = 0x2285;
1417     my $SG_GET_VERSION_NUM = 0x2282;
1418
1419     my $versionbuf = "\x00" x 8;
1420     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1421     if (!$ret) {
1422         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1423         return undef;
1424     }
1425     my $version = unpack("I", $versionbuf);
1426     if ($version < 30000) {
1427         die "scsi generic interface too old\n"  if !$noerr;
1428         return undef;
1429     }
1430
1431     my $buf = "\x00" x 36;
1432     my $sensebuf = "\x00" x 8;
1433     my $cmd = pack("C x3 C x1", 0x12, 36);
1434
1435     # see /usr/include/scsi/sg.h
1436     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1437
1438     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1439                       length($sensebuf), 0, length($buf), $buf,
1440                       $cmd, $sensebuf, 6000);
1441
1442     $ret = ioctl($fh, $SG_IO, $packet);
1443     if (!$ret) {
1444         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1445         return undef;
1446     }
1447
1448     my @res = unpack($sg_io_hdr_t, $packet);
1449     if ($res[17] || $res[18]) {
1450         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1451         return undef;
1452     }
1453
1454     my $res = {};
1455     (my $byte0, my $byte1, $res->{vendor},
1456      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1457
1458     $res->{removable} = $byte1 & 128 ? 1 : 0;
1459     $res->{type} = $byte0 & 31;
1460
1461     return $res;
1462 }
1463
1464 sub path_is_scsi {
1465     my ($path) = @_;
1466
1467     my $fh = IO::File->new("+<$path") || return undef;
1468     my $res = scsi_inquiry($fh, 1);
1469     close($fh);
1470
1471     return $res;
1472 }
1473
1474 sub machine_type_is_q35 {
1475     my ($conf) = @_;
1476
1477     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1478 }
1479
1480 sub print_tabletdevice_full {
1481     my ($conf) = @_;
1482
1483     my $q35 = machine_type_is_q35($conf);
1484
1485     # we use uhci for old VMs because tablet driver was buggy in older qemu
1486     my $usbbus = $q35 ? "ehci" : "uhci";
1487
1488     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1489 }
1490
1491 sub print_drivedevice_full {
1492     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1493
1494     my $device = '';
1495     my $maxdev = 0;
1496
1497     if ($drive->{interface} eq 'virtio') {
1498         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1499         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1500         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1501     } elsif ($drive->{interface} eq 'scsi') {
1502
1503         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1504         my $unit = $drive->{index} % $maxdev;
1505         my $devicetype = 'hd';
1506         my $path = '';
1507         if (drive_is_cdrom($drive)) {
1508             $devicetype = 'cd';
1509         } else {
1510             if ($drive->{file} =~ m|^/|) {
1511                 $path = $drive->{file};
1512                 if (my $info = path_is_scsi($path)) {
1513                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1514                         $devicetype = 'block';
1515                     } elsif ($info->{type} == 1) { # tape
1516                         $devicetype = 'generic';
1517                     }
1518                 }
1519             } else {
1520                  $path = PVE::Storage::path($storecfg, $drive->{file});
1521             }
1522
1523             if($path =~ m/^iscsi\:\/\//){
1524                 $devicetype = 'generic';
1525             }
1526         }
1527
1528         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1529             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1530         } else {
1531             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1532         }
1533
1534     } elsif ($drive->{interface} eq 'ide'){
1535         $maxdev = 2;
1536         my $controller = int($drive->{index} / $maxdev);
1537         my $unit = $drive->{index} % $maxdev;
1538         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1539
1540         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1541         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1542             $model = URI::Escape::uri_unescape($model);
1543             $device .= ",model=$model";
1544         }
1545     } elsif ($drive->{interface} eq 'sata'){
1546         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1547         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1548         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1549     } elsif ($drive->{interface} eq 'usb') {
1550         die "implement me";
1551         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1552     } else {
1553         die "unsupported interface type";
1554     }
1555
1556     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1557
1558     return $device;
1559 }
1560
1561 sub get_initiator_name {
1562     my $initiator;
1563
1564     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1565     while (defined(my $line = <$fh>)) {
1566         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1567         $initiator = $1;
1568         last;
1569     }
1570     $fh->close();
1571
1572     return $initiator;
1573 }
1574
1575 sub print_drive_full {
1576     my ($storecfg, $vmid, $drive) = @_;
1577
1578     my $path;
1579     my $volid = $drive->{file};
1580     my $format;
1581
1582     if (drive_is_cdrom($drive)) {
1583         $path = get_iso_path($storecfg, $vmid, $volid);
1584     } else {
1585         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1586         if ($storeid) {
1587             $path = PVE::Storage::path($storecfg, $volid);
1588             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1589             $format = qemu_img_format($scfg, $volname);
1590         } else {
1591             $path = $volid;
1592             $format = "raw";
1593         }
1594    }
1595
1596     my $opts = '';
1597     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard);
1598     foreach my $o (@qemu_drive_options) {
1599         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1600     }
1601     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1602         my ($dir, $qmpname) = @$type;
1603         if (my $v = $drive->{"mbps$dir"}) {
1604             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1605         }
1606         if (my $v = $drive->{"mbps${dir}_max"}) {
1607             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1608         }
1609         if (my $v = $drive->{"bps${dir}_max_length"}) {
1610             $opts .= ",throttling.bps$qmpname-max-length=$v";
1611         }
1612         if (my $v = $drive->{"iops${dir}"}) {
1613             $opts .= ",throttling.iops$qmpname=$v";
1614         }
1615         if (my $v = $drive->{"iops${dir}_max"}) {
1616             $opts .= ",throttling.iops$qmpname=-max$v";
1617         }
1618         if (my $v = $drive->{"iops${dir}_max_length"}) {
1619             $opts .= ",throttling.iops$qmpname=-max-length$v";
1620         }
1621     }
1622
1623     if (my $serial = $drive->{serial}) {
1624         $serial = URI::Escape::uri_unescape($serial);
1625         $opts .= ",serial=$serial";
1626     }
1627
1628     $opts .= ",format=$format" if $format && !$drive->{format};
1629
1630     my $cache_direct = 0;
1631
1632     if (my $cache = $drive->{cache}) {
1633         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1634     } elsif (!drive_is_cdrom($drive)) {
1635         $opts .= ",cache=none";
1636         $cache_direct = 1;
1637     }
1638
1639     # aio native works only with O_DIRECT
1640     if (!$drive->{aio}) {
1641         if($cache_direct) {
1642             $opts .= ",aio=native";
1643         } else {
1644             $opts .= ",aio=threads";
1645         }
1646     }
1647
1648     if (!drive_is_cdrom($drive)) {
1649         my $detectzeroes;
1650         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1651             $detectzeroes = 'off';
1652         } elsif ($drive->{discard}) {
1653             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1654         } else {
1655             # This used to be our default with discard not being specified:
1656             $detectzeroes = 'on';
1657         }
1658         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1659     }
1660
1661     my $pathinfo = $path ? "file=$path," : '';
1662
1663     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1664 }
1665
1666 sub print_netdevice_full {
1667     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1668
1669     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1670
1671     my $device = $net->{model};
1672     if ($net->{model} eq 'virtio') {
1673          $device = 'virtio-net-pci';
1674      };
1675
1676     my $pciaddr = print_pci_addr("$netid", $bridges);
1677     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1678     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1679         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1680         my $vectors = $net->{queues} * 2 + 2;
1681         $tmpstr .= ",vectors=$vectors,mq=on";
1682     }
1683     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1684
1685     if ($use_old_bios_files) {
1686         my $romfile;
1687         if ($device eq 'virtio-net-pci') {
1688             $romfile = 'pxe-virtio.rom';
1689         } elsif ($device eq 'e1000') {
1690             $romfile = 'pxe-e1000.rom';
1691         } elsif ($device eq 'ne2k') {
1692             $romfile = 'pxe-ne2k_pci.rom';
1693         } elsif ($device eq 'pcnet') {
1694             $romfile = 'pxe-pcnet.rom';
1695         } elsif ($device eq 'rtl8139') {
1696             $romfile = 'pxe-rtl8139.rom';
1697         }
1698         $tmpstr .= ",romfile=$romfile" if $romfile;
1699     }
1700
1701     return $tmpstr;
1702 }
1703
1704 sub print_netdev_full {
1705     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1706
1707     my $i = '';
1708     if ($netid =~ m/^net(\d+)$/) {
1709         $i = int($1);
1710     }
1711
1712     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1713
1714     my $ifname = "tap${vmid}i$i";
1715
1716     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1717     die "interface name '$ifname' is too long (max 15 character)\n"
1718         if length($ifname) >= 16;
1719
1720     my $vhostparam = '';
1721     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1722
1723     my $vmname = $conf->{name} || "vm$vmid";
1724
1725     my $netdev = "";
1726     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1727
1728     if ($net->{bridge}) {
1729         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1730     } else {
1731         $netdev = "type=user,id=$netid,hostname=$vmname";
1732     }
1733
1734     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1735
1736     return $netdev;
1737 }
1738
1739
1740 sub print_cpu_device {
1741     my ($conf, $id) = @_;
1742
1743     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
1744     my $cpu = $nokvm ? "qemu64" : "kvm64";
1745     if (my $cputype = $conf->{cpu}) {
1746         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1747             or die "Cannot parse cpu description: $cputype\n";
1748         $cpu = $cpuconf->{cputype};
1749     }
1750
1751     my $cores = $conf->{cores} || 1;
1752
1753     my $current_core = ($id - 1) % $cores;
1754     my $current_socket = int(($id - 1 - $current_core)/$cores);
1755
1756     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1757 }
1758
1759 sub drive_is_cdrom {
1760     my ($drive) = @_;
1761
1762     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1763
1764 }
1765
1766 sub parse_number_sets {
1767     my ($set) = @_;
1768     my $res = [];
1769     foreach my $part (split(/;/, $set)) {
1770         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1771             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1772             push @$res, [ $1, $2 ];
1773         } else {
1774             die "invalid range: $part\n";
1775         }
1776     }
1777     return $res;
1778 }
1779
1780 sub parse_numa {
1781     my ($data) = @_;
1782
1783     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1784     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1785     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1786     return $res;
1787 }
1788
1789 sub parse_hostpci {
1790     my ($value) = @_;
1791
1792     return undef if !$value;
1793
1794     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1795
1796     my @idlist = split(/;/, $res->{host});
1797     delete $res->{host};
1798     foreach my $id (@idlist) {
1799         if ($id =~ /^$PCIRE$/) {
1800             if (defined($2)) {
1801                 push @{$res->{pciid}}, { id => $1, function => $2 };
1802             } else {
1803                 my $pcidevices = lspci($1);
1804                 $res->{pciid} = $pcidevices->{$1};
1805             }
1806         } else {
1807             # should have been caught by parse_property_string already
1808             die "failed to parse PCI id: $id\n";
1809         }
1810     }
1811     return $res;
1812 }
1813
1814 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1815 sub parse_net {
1816     my ($data) = @_;
1817
1818     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1819     if ($@) {
1820         warn $@;
1821         return undef;
1822     }
1823     if (!defined($res->{macaddr})) {
1824         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1825         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1826     }
1827     return $res;
1828 }
1829
1830 sub print_net {
1831     my $net = shift;
1832
1833     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1834 }
1835
1836 sub add_random_macs {
1837     my ($settings) = @_;
1838
1839     foreach my $opt (keys %$settings) {
1840         next if $opt !~ m/^net(\d+)$/;
1841         my $net = parse_net($settings->{$opt});
1842         next if !$net;
1843         $settings->{$opt} = print_net($net);
1844     }
1845 }
1846
1847 sub vm_is_volid_owner {
1848     my ($storecfg, $vmid, $volid) = @_;
1849
1850     if ($volid !~  m|^/|) {
1851         my ($path, $owner);
1852         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1853         if ($owner && ($owner == $vmid)) {
1854             return 1;
1855         }
1856     }
1857
1858     return undef;
1859 }
1860
1861 sub split_flagged_list {
1862     my $text = shift || '';
1863     $text =~ s/[,;]/ /g;
1864     $text =~ s/^\s+//;
1865     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1866 }
1867
1868 sub join_flagged_list {
1869     my ($how, $lst) = @_;
1870     join $how, map { $lst->{$_} . $_ } keys %$lst;
1871 }
1872
1873 sub vmconfig_delete_pending_option {
1874     my ($conf, $key, $force) = @_;
1875
1876     delete $conf->{pending}->{$key};
1877     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1878     $pending_delete_hash->{$key} = $force ? '!' : '';
1879     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1880 }
1881
1882 sub vmconfig_undelete_pending_option {
1883     my ($conf, $key) = @_;
1884
1885     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1886     delete $pending_delete_hash->{$key};
1887
1888     if (%$pending_delete_hash) {
1889         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1890     } else {
1891         delete $conf->{pending}->{delete};
1892     }
1893 }
1894
1895 sub vmconfig_register_unused_drive {
1896     my ($storecfg, $vmid, $conf, $drive) = @_;
1897
1898     if (!drive_is_cdrom($drive)) {
1899         my $volid = $drive->{file};
1900         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1901             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1902         }
1903     }
1904 }
1905
1906 sub vmconfig_cleanup_pending {
1907     my ($conf) = @_;
1908
1909     # remove pending changes when nothing changed
1910     my $changes;
1911     foreach my $opt (keys %{$conf->{pending}}) {
1912         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1913             $changes = 1;
1914             delete $conf->{pending}->{$opt};
1915         }
1916     }
1917
1918     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1919     my $pending_delete_hash = {};
1920     while (my ($opt, $force) = each %$current_delete_hash) {
1921         if (defined($conf->{$opt})) {
1922             $pending_delete_hash->{$opt} = $force;
1923         } else {
1924             $changes = 1;
1925         }
1926     }
1927
1928     if (%$pending_delete_hash) {
1929         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1930     } else {
1931         delete $conf->{pending}->{delete};
1932     }
1933
1934     return $changes;
1935 }
1936
1937 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1938 my $smbios1_fmt = {
1939     uuid => {
1940         type => 'string',
1941         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1942         format_description => 'UUID',
1943         description => "Set SMBIOS1 UUID.",
1944         optional => 1,
1945     },
1946     version => {
1947         type => 'string',
1948         pattern => '\S+',
1949         format_description => 'string',
1950         description => "Set SMBIOS1 version.",
1951         optional => 1,
1952     },
1953     serial => {
1954         type => 'string',
1955         pattern => '\S+',
1956         format_description => 'string',
1957         description => "Set SMBIOS1 serial number.",
1958         optional => 1,
1959     },
1960     manufacturer => {
1961         type => 'string',
1962         pattern => '\S+',
1963         format_description => 'string',
1964         description => "Set SMBIOS1 manufacturer.",
1965         optional => 1,
1966     },
1967     product => {
1968         type => 'string',
1969         pattern => '\S+',
1970         format_description => 'string',
1971         description => "Set SMBIOS1 product ID.",
1972         optional => 1,
1973     },
1974     sku => {
1975         type => 'string',
1976         pattern => '\S+',
1977         format_description => 'string',
1978         description => "Set SMBIOS1 SKU string.",
1979         optional => 1,
1980     },
1981     family => {
1982         type => 'string',
1983         pattern => '\S+',
1984         format_description => 'string',
1985         description => "Set SMBIOS1 family string.",
1986         optional => 1,
1987     },
1988 };
1989
1990 sub parse_smbios1 {
1991     my ($data) = @_;
1992
1993     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1994     warn $@ if $@;
1995     return $res;
1996 }
1997
1998 sub print_smbios1 {
1999     my ($smbios1) = @_;
2000     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2001 }
2002
2003 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2004
2005 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2006 sub verify_bootdisk {
2007     my ($value, $noerr) = @_;
2008
2009     return $value if is_valid_drivename($value);
2010
2011     return undef if $noerr;
2012
2013     die "invalid boot disk '$value'\n";
2014 }
2015
2016 sub parse_watchdog {
2017     my ($value) = @_;
2018
2019     return undef if !$value;
2020
2021     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2022     warn $@ if $@;
2023     return $res;
2024 }
2025
2026 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2027 sub verify_usb_device {
2028     my ($value, $noerr) = @_;
2029
2030     return $value if parse_usb_device($value);
2031
2032     return undef if $noerr;
2033
2034     die "unable to parse usb device\n";
2035 }
2036
2037 # add JSON properties for create and set function
2038 sub json_config_properties {
2039     my $prop = shift;
2040
2041     foreach my $opt (keys %$confdesc) {
2042         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2043         $prop->{$opt} = $confdesc->{$opt};
2044     }
2045
2046     return $prop;
2047 }
2048
2049 sub check_type {
2050     my ($key, $value) = @_;
2051
2052     die "unknown setting '$key'\n" if !$confdesc->{$key};
2053
2054     my $type = $confdesc->{$key}->{type};
2055
2056     if (!defined($value)) {
2057         die "got undefined value\n";
2058     }
2059
2060     if ($value =~ m/[\n\r]/) {
2061         die "property contains a line feed\n";
2062     }
2063
2064     if ($type eq 'boolean') {
2065         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2066         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2067         die "type check ('boolean') failed - got '$value'\n";
2068     } elsif ($type eq 'integer') {
2069         return int($1) if $value =~ m/^(\d+)$/;
2070         die "type check ('integer') failed - got '$value'\n";
2071     } elsif ($type eq 'number') {
2072         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2073         die "type check ('number') failed - got '$value'\n";
2074     } elsif ($type eq 'string') {
2075         if (my $fmt = $confdesc->{$key}->{format}) {
2076             PVE::JSONSchema::check_format($fmt, $value);
2077             return $value;
2078         }
2079         $value =~ s/^\"(.*)\"$/$1/;
2080         return $value;
2081     } else {
2082         die "internal error"
2083     }
2084 }
2085
2086 sub check_iommu_support{
2087     #fixme : need to check IOMMU support
2088     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2089
2090     my $iommu=1;
2091     return $iommu;
2092
2093 }
2094
2095 sub touch_config {
2096     my ($vmid) = @_;
2097
2098     my $conf = PVE::QemuConfig->config_file($vmid);
2099     utime undef, undef, $conf;
2100 }
2101
2102 sub destroy_vm {
2103     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2104
2105     my $conffile = PVE::QemuConfig->config_file($vmid);
2106
2107     my $conf = PVE::QemuConfig->load_config($vmid);
2108
2109     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2110
2111     # only remove disks owned by this VM
2112     foreach_drive($conf, sub {
2113         my ($ds, $drive) = @_;
2114
2115         return if drive_is_cdrom($drive);
2116
2117         my $volid = $drive->{file};
2118
2119         return if !$volid || $volid =~ m|^/|;
2120
2121         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2122         return if !$path || !$owner || ($owner != $vmid);
2123
2124         eval {
2125             PVE::Storage::vdisk_free($storecfg, $volid);
2126         };
2127         warn "Could not remove disk '$volid', check manually: $@" if $@;
2128
2129     });
2130
2131     if ($keep_empty_config) {
2132         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2133     } else {
2134         unlink $conffile;
2135     }
2136
2137     # also remove unused disk
2138     eval {
2139         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2140
2141         eval {
2142             PVE::Storage::foreach_volid($dl, sub {
2143                 my ($volid, $sid, $volname, $d) = @_;
2144                 PVE::Storage::vdisk_free($storecfg, $volid);
2145             });
2146         };
2147         warn $@ if $@;
2148
2149     };
2150     warn $@ if $@;
2151 }
2152
2153 sub parse_vm_config {
2154     my ($filename, $raw) = @_;
2155
2156     return undef if !defined($raw);
2157
2158     my $res = {
2159         digest => Digest::SHA::sha1_hex($raw),
2160         snapshots => {},
2161         pending => {},
2162     };
2163
2164     $filename =~ m|/qemu-server/(\d+)\.conf$|
2165         || die "got strange filename '$filename'";
2166
2167     my $vmid = $1;
2168
2169     my $conf = $res;
2170     my $descr;
2171     my $section = '';
2172
2173     my @lines = split(/\n/, $raw);
2174     foreach my $line (@lines) {
2175         next if $line =~ m/^\s*$/;
2176
2177         if ($line =~ m/^\[PENDING\]\s*$/i) {
2178             $section = 'pending';
2179             if (defined($descr)) {
2180                 $descr =~ s/\s+$//;
2181                 $conf->{description} = $descr;
2182             }
2183             $descr = undef;
2184             $conf = $res->{$section} = {};
2185             next;
2186
2187         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2188             $section = $1;
2189             if (defined($descr)) {
2190                 $descr =~ s/\s+$//;
2191                 $conf->{description} = $descr;
2192             }
2193             $descr = undef;
2194             $conf = $res->{snapshots}->{$section} = {};
2195             next;
2196         }
2197
2198         if ($line =~ m/^\#(.*)\s*$/) {
2199             $descr = '' if !defined($descr);
2200             $descr .= PVE::Tools::decode_text($1) . "\n";
2201             next;
2202         }
2203
2204         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2205             $descr = '' if !defined($descr);
2206             $descr .= PVE::Tools::decode_text($2);
2207         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2208             $conf->{snapstate} = $1;
2209         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2210             my $key = $1;
2211             my $value = $2;
2212             $conf->{$key} = $value;
2213         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2214             my $value = $1;
2215             if ($section eq 'pending') {
2216                 $conf->{delete} = $value; # we parse this later
2217             } else {
2218                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2219             }
2220         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2221             my $key = $1;
2222             my $value = $2;
2223             eval { $value = check_type($key, $value); };
2224             if ($@) {
2225                 warn "vm $vmid - unable to parse value of '$key' - $@";
2226             } else {
2227                 $key = 'ide2' if $key eq 'cdrom';
2228                 my $fmt = $confdesc->{$key}->{format};
2229                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2230                     my $v = parse_drive($key, $value);
2231                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2232                         $v->{file} = $volid;
2233                         $value = print_drive($vmid, $v);
2234                     } else {
2235                         warn "vm $vmid - unable to parse value of '$key'\n";
2236                         next;
2237                     }
2238                 }
2239
2240                 $conf->{$key} = $value;
2241             }
2242         }
2243     }
2244
2245     if (defined($descr)) {
2246         $descr =~ s/\s+$//;
2247         $conf->{description} = $descr;
2248     }
2249     delete $res->{snapstate}; # just to be sure
2250
2251     return $res;
2252 }
2253
2254 sub write_vm_config {
2255     my ($filename, $conf) = @_;
2256
2257     delete $conf->{snapstate}; # just to be sure
2258
2259     if ($conf->{cdrom}) {
2260         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2261         $conf->{ide2} = $conf->{cdrom};
2262         delete $conf->{cdrom};
2263     }
2264
2265     # we do not use 'smp' any longer
2266     if ($conf->{sockets}) {
2267         delete $conf->{smp};
2268     } elsif ($conf->{smp}) {
2269         $conf->{sockets} = $conf->{smp};
2270         delete $conf->{cores};
2271         delete $conf->{smp};
2272     }
2273
2274     my $used_volids = {};
2275
2276     my $cleanup_config = sub {
2277         my ($cref, $pending, $snapname) = @_;
2278
2279         foreach my $key (keys %$cref) {
2280             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2281                 $key eq 'snapstate' || $key eq 'pending';
2282             my $value = $cref->{$key};
2283             if ($key eq 'delete') {
2284                 die "propertry 'delete' is only allowed in [PENDING]\n"
2285                     if !$pending;
2286                 # fixme: check syntax?
2287                 next;
2288             }
2289             eval { $value = check_type($key, $value); };
2290             die "unable to parse value of '$key' - $@" if $@;
2291
2292             $cref->{$key} = $value;
2293
2294             if (!$snapname && is_valid_drivename($key)) {
2295                 my $drive = parse_drive($key, $value);
2296                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2297             }
2298         }
2299     };
2300
2301     &$cleanup_config($conf);
2302
2303     &$cleanup_config($conf->{pending}, 1);
2304
2305     foreach my $snapname (keys %{$conf->{snapshots}}) {
2306         die "internal error" if $snapname eq 'pending';
2307         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2308     }
2309
2310     # remove 'unusedX' settings if we re-add a volume
2311     foreach my $key (keys %$conf) {
2312         my $value = $conf->{$key};
2313         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2314             delete $conf->{$key};
2315         }
2316     }
2317
2318     my $generate_raw_config = sub {
2319         my ($conf, $pending) = @_;
2320
2321         my $raw = '';
2322
2323         # add description as comment to top of file
2324         if (defined(my $descr = $conf->{description})) {
2325             if ($descr) {
2326                 foreach my $cl (split(/\n/, $descr)) {
2327                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2328                 }
2329             } else {
2330                 $raw .= "#\n" if $pending;
2331             }
2332         }
2333
2334         foreach my $key (sort keys %$conf) {
2335             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2336             $raw .= "$key: $conf->{$key}\n";
2337         }
2338         return $raw;
2339     };
2340
2341     my $raw = &$generate_raw_config($conf);
2342
2343     if (scalar(keys %{$conf->{pending}})){
2344         $raw .= "\n[PENDING]\n";
2345         $raw .= &$generate_raw_config($conf->{pending}, 1);
2346     }
2347
2348     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2349         $raw .= "\n[$snapname]\n";
2350         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2351     }
2352
2353     return $raw;
2354 }
2355
2356 sub load_defaults {
2357
2358     my $res = {};
2359
2360     # we use static defaults from our JSON schema configuration
2361     foreach my $key (keys %$confdesc) {
2362         if (defined(my $default = $confdesc->{$key}->{default})) {
2363             $res->{$key} = $default;
2364         }
2365     }
2366
2367     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2368     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2369
2370     return $res;
2371 }
2372
2373 sub config_list {
2374     my $vmlist = PVE::Cluster::get_vmlist();
2375     my $res = {};
2376     return $res if !$vmlist || !$vmlist->{ids};
2377     my $ids = $vmlist->{ids};
2378
2379     foreach my $vmid (keys %$ids) {
2380         my $d = $ids->{$vmid};
2381         next if !$d->{node} || $d->{node} ne $nodename;
2382         next if !$d->{type} || $d->{type} ne 'qemu';
2383         $res->{$vmid}->{exists} = 1;
2384     }
2385     return $res;
2386 }
2387
2388 # test if VM uses local resources (to prevent migration)
2389 sub check_local_resources {
2390     my ($conf, $noerr) = @_;
2391
2392     my $loc_res = 0;
2393
2394     $loc_res = 1 if $conf->{hostusb}; # old syntax
2395     $loc_res = 1 if $conf->{hostpci}; # old syntax
2396
2397     foreach my $k (keys %$conf) {
2398         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2399         # sockets are safe: they will recreated be on the target side post-migrate
2400         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2401         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2402     }
2403
2404     die "VM uses local resources\n" if $loc_res && !$noerr;
2405
2406     return $loc_res;
2407 }
2408
2409 # check if used storages are available on all nodes (use by migrate)
2410 sub check_storage_availability {
2411     my ($storecfg, $conf, $node) = @_;
2412
2413     foreach_drive($conf, sub {
2414         my ($ds, $drive) = @_;
2415
2416         my $volid = $drive->{file};
2417         return if !$volid;
2418
2419         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2420         return if !$sid;
2421
2422         # check if storage is available on both nodes
2423         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2424         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2425    });
2426 }
2427
2428 # list nodes where all VM images are available (used by has_feature API)
2429 sub shared_nodes {
2430     my ($conf, $storecfg) = @_;
2431
2432     my $nodelist = PVE::Cluster::get_nodelist();
2433     my $nodehash = { map { $_ => 1 } @$nodelist };
2434     my $nodename = PVE::INotify::nodename();
2435
2436     foreach_drive($conf, sub {
2437         my ($ds, $drive) = @_;
2438
2439         my $volid = $drive->{file};
2440         return if !$volid;
2441
2442         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2443         if ($storeid) {
2444             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2445             if ($scfg->{disable}) {
2446                 $nodehash = {};
2447             } elsif (my $avail = $scfg->{nodes}) {
2448                 foreach my $node (keys %$nodehash) {
2449                     delete $nodehash->{$node} if !$avail->{$node};
2450                 }
2451             } elsif (!$scfg->{shared}) {
2452                 foreach my $node (keys %$nodehash) {
2453                     delete $nodehash->{$node} if $node ne $nodename
2454                 }
2455             }
2456         }
2457     });
2458
2459     return $nodehash
2460 }
2461
2462 sub check_cmdline {
2463     my ($pidfile, $pid) = @_;
2464
2465     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2466     if (defined($fh)) {
2467         my $line = <$fh>;
2468         $fh->close;
2469         return undef if !$line;
2470         my @param = split(/\0/, $line);
2471
2472         my $cmd = $param[0];
2473         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2474
2475         for (my $i = 0; $i < scalar (@param); $i++) {
2476             my $p = $param[$i];
2477             next if !$p;
2478             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2479                 my $p = $param[$i+1];
2480                 return 1 if $p && ($p eq $pidfile);
2481                 return undef;
2482             }
2483         }
2484     }
2485     return undef;
2486 }
2487
2488 sub check_running {
2489     my ($vmid, $nocheck, $node) = @_;
2490
2491     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2492
2493     die "unable to find configuration file for VM $vmid - no such machine\n"
2494         if !$nocheck && ! -f $filename;
2495
2496     my $pidfile = pidfile_name($vmid);
2497
2498     if (my $fd = IO::File->new("<$pidfile")) {
2499         my $st = stat($fd);
2500         my $line = <$fd>;
2501         close($fd);
2502
2503         my $mtime = $st->mtime;
2504         if ($mtime > time()) {
2505             warn "file '$filename' modified in future\n";
2506         }
2507
2508         if ($line =~ m/^(\d+)$/) {
2509             my $pid = $1;
2510             if (check_cmdline($pidfile, $pid)) {
2511                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2512                     return $pid;
2513                 }
2514             }
2515         }
2516     }
2517
2518     return undef;
2519 }
2520
2521 sub vzlist {
2522
2523     my $vzlist = config_list();
2524
2525     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2526
2527     while (defined(my $de = $fd->read)) {
2528         next if $de !~ m/^(\d+)\.pid$/;
2529         my $vmid = $1;
2530         next if !defined($vzlist->{$vmid});
2531         if (my $pid = check_running($vmid)) {
2532             $vzlist->{$vmid}->{pid} = $pid;
2533         }
2534     }
2535
2536     return $vzlist;
2537 }
2538
2539 sub disksize {
2540     my ($storecfg, $conf) = @_;
2541
2542     my $bootdisk = $conf->{bootdisk};
2543     return undef if !$bootdisk;
2544     return undef if !is_valid_drivename($bootdisk);
2545
2546     return undef if !$conf->{$bootdisk};
2547
2548     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2549     return undef if !defined($drive);
2550
2551     return undef if drive_is_cdrom($drive);
2552
2553     my $volid = $drive->{file};
2554     return undef if !$volid;
2555
2556     return $drive->{size};
2557 }
2558
2559 my $last_proc_pid_stat;
2560
2561 # get VM status information
2562 # This must be fast and should not block ($full == false)
2563 # We only query KVM using QMP if $full == true (this can be slow)
2564 sub vmstatus {
2565     my ($opt_vmid, $full) = @_;
2566
2567     my $res = {};
2568
2569     my $storecfg = PVE::Storage::config();
2570
2571     my $list = vzlist();
2572     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2573
2574     my $cpucount = $cpuinfo->{cpus} || 1;
2575
2576     foreach my $vmid (keys %$list) {
2577         next if $opt_vmid && ($vmid ne $opt_vmid);
2578
2579         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2580         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2581
2582         my $d = {};
2583         $d->{pid} = $list->{$vmid}->{pid};
2584
2585         # fixme: better status?
2586         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2587
2588         my $size = disksize($storecfg, $conf);
2589         if (defined($size)) {
2590             $d->{disk} = 0; # no info available
2591             $d->{maxdisk} = $size;
2592         } else {
2593             $d->{disk} = 0;
2594             $d->{maxdisk} = 0;
2595         }
2596
2597         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2598         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2599         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2600
2601         $d->{name} = $conf->{name} || "VM $vmid";
2602         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2603
2604         if ($conf->{balloon}) {
2605             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2606             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2607         }
2608
2609         $d->{uptime} = 0;
2610         $d->{cpu} = 0;
2611         $d->{mem} = 0;
2612
2613         $d->{netout} = 0;
2614         $d->{netin} = 0;
2615
2616         $d->{diskread} = 0;
2617         $d->{diskwrite} = 0;
2618
2619         $d->{template} = PVE::QemuConfig->is_template($conf);
2620
2621         $res->{$vmid} = $d;
2622     }
2623
2624     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2625     foreach my $dev (keys %$netdev) {
2626         next if $dev !~ m/^tap([1-9]\d*)i/;
2627         my $vmid = $1;
2628         my $d = $res->{$vmid};
2629         next if !$d;
2630
2631         $d->{netout} += $netdev->{$dev}->{receive};
2632         $d->{netin} += $netdev->{$dev}->{transmit};
2633
2634         if ($full) {
2635             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2636             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2637         }
2638
2639     }
2640
2641     my $ctime = gettimeofday;
2642
2643     foreach my $vmid (keys %$list) {
2644
2645         my $d = $res->{$vmid};
2646         my $pid = $d->{pid};
2647         next if !$pid;
2648
2649         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2650         next if !$pstat; # not running
2651
2652         my $used = $pstat->{utime} + $pstat->{stime};
2653
2654         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2655
2656         if ($pstat->{vsize}) {
2657             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2658         }
2659
2660         my $old = $last_proc_pid_stat->{$pid};
2661         if (!$old) {
2662             $last_proc_pid_stat->{$pid} = {
2663                 time => $ctime,
2664                 used => $used,
2665                 cpu => 0,
2666             };
2667             next;
2668         }
2669
2670         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2671
2672         if ($dtime > 1000) {
2673             my $dutime = $used -  $old->{used};
2674
2675             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2676             $last_proc_pid_stat->{$pid} = {
2677                 time => $ctime,
2678                 used => $used,
2679                 cpu => $d->{cpu},
2680             };
2681         } else {
2682             $d->{cpu} = $old->{cpu};
2683         }
2684     }
2685
2686     return $res if !$full;
2687
2688     my $qmpclient = PVE::QMPClient->new();
2689
2690     my $ballooncb = sub {
2691         my ($vmid, $resp) = @_;
2692
2693         my $info = $resp->{'return'};
2694         return if !$info->{max_mem};
2695
2696         my $d = $res->{$vmid};
2697
2698         # use memory assigned to VM
2699         $d->{maxmem} = $info->{max_mem};
2700         $d->{balloon} = $info->{actual};
2701
2702         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2703             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2704             $d->{freemem} = $info->{free_mem};
2705         }
2706
2707         $d->{ballooninfo} = $info;
2708     };
2709
2710     my $blockstatscb = sub {
2711         my ($vmid, $resp) = @_;
2712         my $data = $resp->{'return'} || [];
2713         my $totalrdbytes = 0;
2714         my $totalwrbytes = 0;
2715
2716         for my $blockstat (@$data) {
2717             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2718             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2719
2720             $blockstat->{device} =~ s/drive-//;
2721             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2722         }
2723         $res->{$vmid}->{diskread} = $totalrdbytes;
2724         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2725     };
2726
2727     my $statuscb = sub {
2728         my ($vmid, $resp) = @_;
2729
2730         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2731         # this fails if ballon driver is not loaded, so this must be
2732         # the last commnand (following command are aborted if this fails).
2733         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2734
2735         my $status = 'unknown';
2736         if (!defined($status = $resp->{'return'}->{status})) {
2737             warn "unable to get VM status\n";
2738             return;
2739         }
2740
2741         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2742     };
2743
2744     foreach my $vmid (keys %$list) {
2745         next if $opt_vmid && ($vmid ne $opt_vmid);
2746         next if !$res->{$vmid}->{pid}; # not running
2747         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2748     }
2749
2750     $qmpclient->queue_execute(undef, 2);
2751
2752     foreach my $vmid (keys %$list) {
2753         next if $opt_vmid && ($vmid ne $opt_vmid);
2754         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2755     }
2756
2757     return $res;
2758 }
2759
2760 sub foreach_drive {
2761     my ($conf, $func, @param) = @_;
2762
2763     foreach my $ds (valid_drive_names()) {
2764         next if !defined($conf->{$ds});
2765
2766         my $drive = parse_drive($ds, $conf->{$ds});
2767         next if !$drive;
2768
2769         &$func($ds, $drive, @param);
2770     }
2771 }
2772
2773 sub foreach_volid {
2774     my ($conf, $func, @param) = @_;
2775
2776     my $volhash = {};
2777
2778     my $test_volid = sub {
2779         my ($volid, $is_cdrom) = @_;
2780
2781         return if !$volid;
2782
2783         $volhash->{$volid} = $is_cdrom || 0;
2784     };
2785
2786     foreach_drive($conf, sub {
2787         my ($ds, $drive) = @_;
2788         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2789     });
2790
2791     foreach my $snapname (keys %{$conf->{snapshots}}) {
2792         my $snap = $conf->{snapshots}->{$snapname};
2793         &$test_volid($snap->{vmstate}, 0);
2794         foreach_drive($snap, sub {
2795             my ($ds, $drive) = @_;
2796             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2797         });
2798     }
2799
2800     foreach my $volid (keys %$volhash) {
2801         &$func($volid, $volhash->{$volid}, @param);
2802     }
2803 }
2804
2805 sub vga_conf_has_spice {
2806     my ($vga) = @_;
2807
2808     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2809
2810     return $1 || 1;
2811 }
2812
2813 sub config_to_command {
2814     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2815
2816     my $cmd = [];
2817     my $globalFlags = [];
2818     my $machineFlags = [];
2819     my $rtcFlags = [];
2820     my $cpuFlags = [];
2821     my $devices = [];
2822     my $pciaddr = '';
2823     my $bridges = {};
2824     my $kvmver = kvm_user_version();
2825     my $vernum = 0; # unknown
2826     my $ostype = $conf->{ostype};
2827     my $winversion = windows_version($ostype);
2828
2829     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2830         $vernum = $1*1000000+$2*1000;
2831     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2832         $vernum = $1*1000000+$2*1000+$3;
2833     }
2834
2835     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2836
2837     my $have_ovz = -f '/proc/vz/vestat';
2838
2839     my $q35 = machine_type_is_q35($conf);
2840     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2841     my $machine_type = $forcemachine || $conf->{machine};
2842     my $use_old_bios_files = undef;
2843     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2844
2845     my $cpuunits = defined($conf->{cpuunits}) ?
2846             $conf->{cpuunits} : $defaults->{cpuunits};
2847
2848     push @$cmd, '/usr/bin/kvm';
2849
2850     push @$cmd, '-id', $vmid;
2851
2852     my $use_virtio = 0;
2853
2854     my $qmpsocket = qmp_socket($vmid);
2855     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2856     push @$cmd, '-mon', "chardev=qmp,mode=control";
2857
2858
2859     push @$cmd, '-pidfile' , pidfile_name($vmid);
2860
2861     push @$cmd, '-daemonize';
2862
2863     if ($conf->{smbios1}) {
2864         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2865     }
2866
2867     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2868         my $ovmfbase;
2869
2870         # prefer the OVMF_CODE variant
2871         if (-f $OVMF_CODE) {
2872             $ovmfbase = $OVMF_CODE;
2873         } elsif (-f $OVMF_IMG) {
2874             $ovmfbase = $OVMF_IMG;
2875         }
2876
2877         die "no uefi base img found\n" if !$ovmfbase;
2878         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmfbase";
2879
2880         if (defined($conf->{efidisk0}) && ($ovmfbase eq $OVMF_CODE)) {
2881             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $conf->{efidisk0});
2882             my $format = $d->{format} // 'raw';
2883             my $path;
2884             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2885             if ($storeid) {
2886                 $path = PVE::Storage::path($storecfg, $d->{file});
2887                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2888                 $format = qemu_img_format($scfg, $volname);
2889             } else {
2890                 $path = $d->{file};
2891                 $format = "raw";
2892             }
2893             push @$cmd, '-drive', "if=pflash,unit=1,id=drive-efidisk0,format=$format,file=$path";
2894         } elsif ($ovmfbase eq $OVMF_CODE) {
2895             warn "using uefi without permanent efivars disk\n";
2896             my $ovmfvar_dst = "/tmp/$vmid-ovmf.fd";
2897             PVE::Tools::file_copy($OVMF_VARS, $ovmfvar_dst, 256*1024);
2898             push @$cmd, '-drive', "if=pflash,unit=1,format=raw,file=$ovmfvar_dst";
2899         } else {
2900             # if the base img is not OVMF_CODE, we do not have to bother
2901             # to create/use a vars image, since it will not be used anyway
2902             # this can only happen if someone manually deletes the OVMF_CODE image
2903             # or has an old pve-qemu-kvm version installed.
2904             # both should not happen, but we ignore it here
2905         }
2906     }
2907
2908
2909     # add usb controllers
2910     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2911     push @$devices, @usbcontrollers if @usbcontrollers;
2912     my $vga = $conf->{vga};
2913
2914     my $qxlnum = vga_conf_has_spice($vga);
2915     $vga = 'qxl' if $qxlnum;
2916
2917     if (!$vga) {
2918         $vga = $winversion >= 6 ? 'std' : 'cirrus';
2919     }
2920
2921     # enable absolute mouse coordinates (needed by vnc)
2922     my $tablet;
2923     if (defined($conf->{tablet})) {
2924         $tablet = $conf->{tablet};
2925     } else {
2926         $tablet = $defaults->{tablet};
2927         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2928         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2929     }
2930
2931     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2932
2933     my $kvm_off = 0;
2934     my $gpu_passthrough;
2935
2936     # host pci devices
2937     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2938         my $d = parse_hostpci($conf->{"hostpci$i"});
2939         next if !$d;
2940
2941         my $pcie = $d->{pcie};
2942         if($pcie){
2943             die "q35 machine model is not enabled" if !$q35;
2944             $pciaddr = print_pcie_addr("hostpci$i");
2945         }else{
2946             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2947         }
2948
2949         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2950         my $romfile = $d->{romfile};
2951
2952         my $xvga = '';
2953         if ($d->{'x-vga'}) {
2954             $xvga = ',x-vga=on';
2955             $kvm_off = 1;
2956             $vga = 'none';
2957             $gpu_passthrough = 1;
2958
2959             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2960                 $xvga = "";
2961             }
2962         }
2963         my $pcidevices = $d->{pciid};
2964         my $multifunction = 1 if @$pcidevices > 1;
2965
2966         my $j=0;
2967         foreach my $pcidevice (@$pcidevices) {
2968
2969             my $id = "hostpci$i";
2970             $id .= ".$j" if $multifunction;
2971             my $addr = $pciaddr;
2972             $addr .= ".$j" if $multifunction;
2973             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2974
2975             if($j == 0){
2976                 $devicestr .= "$rombar$xvga";
2977                 $devicestr .= ",multifunction=on" if $multifunction;
2978                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
2979             }
2980
2981             push @$devices, '-device', $devicestr;
2982             $j++;
2983         }
2984     }
2985
2986     # usb devices
2987     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
2988     push @$devices, @usbdevices if @usbdevices;
2989     # serial devices
2990     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2991         if (my $path = $conf->{"serial$i"}) {
2992             if ($path eq 'socket') {
2993                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2994                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2995                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2996             } else {
2997                 die "no such serial device\n" if ! -c $path;
2998                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2999                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3000             }
3001         }
3002     }
3003
3004     # parallel devices
3005     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3006         if (my $path = $conf->{"parallel$i"}) {
3007             die "no such parallel device\n" if ! -c $path;
3008             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3009             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3010             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3011         }
3012     }
3013
3014     my $vmname = $conf->{name} || "vm$vmid";
3015
3016     push @$cmd, '-name', $vmname;
3017
3018     my $sockets = 1;
3019     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3020     $sockets = $conf->{sockets} if  $conf->{sockets};
3021
3022     my $cores = $conf->{cores} || 1;
3023
3024     my $maxcpus = $sockets * $cores;
3025
3026     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3027
3028     my $allowed_vcpus = $cpuinfo->{cpus};
3029
3030     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3031         if ($allowed_vcpus < $maxcpus);
3032
3033     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3034
3035         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3036         for (my $i = 2; $i <= $vcpus; $i++)  {
3037             my $cpustr = print_cpu_device($conf,$i);
3038             push @$cmd, '-device', $cpustr;
3039         }
3040
3041     } else {
3042
3043         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3044     }
3045     push @$cmd, '-nodefaults';
3046
3047     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3048
3049     my $bootindex_hash = {};
3050     my $i = 1;
3051     foreach my $o (split(//, $bootorder)) {
3052         $bootindex_hash->{$o} = $i*100;
3053         $i++;
3054     }
3055
3056     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3057
3058     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3059
3060     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3061
3062     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3063
3064     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3065         my $socket = vnc_socket($vmid);
3066         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3067     } else {
3068         push @$cmd, '-nographic';
3069     }
3070
3071     # time drift fix
3072     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3073
3074     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3075     my $useLocaltime = $conf->{localtime};
3076
3077     if ($winversion >= 5) { # windows
3078         $useLocaltime = 1 if !defined($conf->{localtime});
3079
3080         # use time drift fix when acpi is enabled
3081         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3082             $tdf = 1 if !defined($conf->{tdf});
3083         }
3084     }
3085
3086     if ($winversion >= 6) {
3087         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3088         push @$cmd, '-no-hpet';
3089     }
3090
3091     push @$rtcFlags, 'driftfix=slew' if $tdf;
3092
3093     if ($nokvm) {
3094         push @$machineFlags, 'accel=tcg';
3095     } else {
3096         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3097     }
3098
3099     if ($machine_type) {
3100         push @$machineFlags, "type=${machine_type}";
3101     }
3102
3103     if ($conf->{startdate}) {
3104         push @$rtcFlags, "base=$conf->{startdate}";
3105     } elsif ($useLocaltime) {
3106         push @$rtcFlags, 'base=localtime';
3107     }
3108
3109     my $cpu = $nokvm ? "qemu64" : "kvm64";
3110     if (my $cputype = $conf->{cpu}) {
3111         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3112             or die "Cannot parse cpu description: $cputype\n";
3113         $cpu = $cpuconf->{cputype};
3114         $kvm_off = 1 if $cpuconf->{hidden};
3115     }
3116
3117     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3118
3119     push @$cpuFlags , '-x2apic'
3120         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3121
3122     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3123
3124     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3125
3126     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3127
3128         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3129         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3130     }
3131
3132     add_hyperv_enlighments($cpuFlags, $winversion, $machine_type, $kvmver, $nokvm, $conf->{bios}, $gpu_passthrough);
3133
3134     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3135
3136     push @$cpuFlags, 'kvm=off' if $kvm_off;
3137
3138     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3139         die "internal error"; # should not happen
3140
3141     push @$cpuFlags, "vendor=${cpu_vendor}"
3142         if $cpu_vendor ne 'default';
3143
3144     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3145
3146     push @$cmd, '-cpu', $cpu;
3147
3148     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3149
3150     push @$cmd, '-S' if $conf->{freeze};
3151
3152     # set keyboard layout
3153     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3154     push @$cmd, '-k', $kb if $kb;
3155
3156     # enable sound
3157     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3158     #push @$cmd, '-soundhw', 'es1370';
3159     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3160
3161     if($conf->{agent}) {
3162         my $qgasocket = qmp_socket($vmid, 1);
3163         my $pciaddr = print_pci_addr("qga0", $bridges);
3164         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3165         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3166         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3167     }
3168
3169     my $spice_port;
3170
3171     if ($qxlnum) {
3172         if ($qxlnum > 1) {
3173             if ($winversion){
3174                 for(my $i = 1; $i < $qxlnum; $i++){
3175                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3176                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3177                 }
3178             } else {
3179                 # assume other OS works like Linux
3180                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3181                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3182             }
3183         }
3184
3185         my $pciaddr = print_pci_addr("spice", $bridges);
3186
3187         my $nodename = PVE::INotify::nodename();
3188         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3189         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3190         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3191         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3192         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3193
3194         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3195
3196         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3197         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3198         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3199     }
3200
3201     # enable balloon by default, unless explicitly disabled
3202     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3203         $pciaddr = print_pci_addr("balloon0", $bridges);
3204         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3205     }
3206
3207     if ($conf->{watchdog}) {
3208         my $wdopts = parse_watchdog($conf->{watchdog});
3209         $pciaddr = print_pci_addr("watchdog", $bridges);
3210         my $watchdog = $wdopts->{model} || 'i6300esb';
3211         push @$devices, '-device', "$watchdog$pciaddr";
3212         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3213     }
3214
3215     my $vollist = [];
3216     my $scsicontroller = {};
3217     my $ahcicontroller = {};
3218     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3219
3220     # Add iscsi initiator name if available
3221     if (my $initiator = get_initiator_name()) {
3222         push @$devices, '-iscsi', "initiator-name=$initiator";
3223     }
3224
3225     foreach_drive($conf, sub {
3226         my ($ds, $drive) = @_;
3227
3228         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3229             push @$vollist, $drive->{file};
3230         }
3231
3232         $use_virtio = 1 if $ds =~ m/^virtio/;
3233
3234         if (drive_is_cdrom ($drive)) {
3235             if ($bootindex_hash->{d}) {
3236                 $drive->{bootindex} = $bootindex_hash->{d};
3237                 $bootindex_hash->{d} += 1;
3238             }
3239         } else {
3240             if ($bootindex_hash->{c}) {
3241                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3242                 $bootindex_hash->{c} += 1;
3243             }
3244         }
3245
3246         if($drive->{interface} eq 'virtio'){
3247            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3248         }
3249
3250         if ($drive->{interface} eq 'scsi') {
3251
3252             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3253
3254             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3255             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3256
3257             my $iothread = '';
3258             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3259                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3260                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3261             } elsif ($drive->{iothread}) {
3262                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3263             }
3264
3265             my $queues = '';
3266             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3267                 $queues = ",num_queues=$drive->{queues}";
3268             }
3269
3270             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3271             $scsicontroller->{$controller}=1;
3272         }
3273
3274         if ($drive->{interface} eq 'sata') {
3275            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3276            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3277            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3278            $ahcicontroller->{$controller}=1;
3279         }
3280
3281         if ($drive->{interface} eq 'efidisk') {
3282             # this will be added somewhere else
3283             return;
3284         }
3285
3286         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3287         push @$devices, '-drive',$drive_cmd;
3288         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3289     });
3290
3291     for (my $i = 0; $i < $MAX_NETS; $i++) {
3292          next if !$conf->{"net$i"};
3293          my $d = parse_net($conf->{"net$i"});
3294          next if !$d;
3295
3296          $use_virtio = 1 if $d->{model} eq 'virtio';
3297
3298          if ($bootindex_hash->{n}) {
3299             $d->{bootindex} = $bootindex_hash->{n};
3300             $bootindex_hash->{n} += 1;
3301          }
3302
3303          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3304          push @$devices, '-netdev', $netdevfull;
3305
3306          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3307          push @$devices, '-device', $netdevicefull;
3308     }
3309
3310     if (!$q35) {
3311         # add pci bridges
3312         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3313            $bridges->{1} = 1;
3314            $bridges->{2} = 1;
3315         }
3316
3317         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3318
3319         while (my ($k, $v) = each %$bridges) {
3320             $pciaddr = print_pci_addr("pci.$k");
3321             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3322         }
3323     }
3324
3325     # add custom args
3326     if ($conf->{args}) {
3327         my $aa = PVE::Tools::split_args($conf->{args});
3328         push @$cmd, @$aa;
3329     }
3330
3331     push @$cmd, @$devices;
3332     push @$cmd, '-rtc', join(',', @$rtcFlags)
3333         if scalar(@$rtcFlags);
3334     push @$cmd, '-machine', join(',', @$machineFlags)
3335         if scalar(@$machineFlags);
3336     push @$cmd, '-global', join(',', @$globalFlags)
3337         if scalar(@$globalFlags);
3338
3339     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3340 }
3341
3342 sub vnc_socket {
3343     my ($vmid) = @_;
3344     return "${var_run_tmpdir}/$vmid.vnc";
3345 }
3346
3347 sub spice_port {
3348     my ($vmid) = @_;
3349
3350     my $res = vm_mon_cmd($vmid, 'query-spice');
3351
3352     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3353 }
3354
3355 sub qmp_socket {
3356     my ($vmid, $qga) = @_;
3357     my $sockettype = $qga ? 'qga' : 'qmp';
3358     return "${var_run_tmpdir}/$vmid.$sockettype";
3359 }
3360
3361 sub pidfile_name {
3362     my ($vmid) = @_;
3363     return "${var_run_tmpdir}/$vmid.pid";
3364 }
3365
3366 sub vm_devices_list {
3367     my ($vmid) = @_;
3368
3369     my $res = vm_mon_cmd($vmid, 'query-pci');
3370     my $devices = {};
3371     foreach my $pcibus (@$res) {
3372         foreach my $device (@{$pcibus->{devices}}) {
3373             next if !$device->{'qdev_id'};
3374             if ($device->{'pci_bridge'}) {
3375                 $devices->{$device->{'qdev_id'}} = 1;
3376                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3377                     next if !$bridge_device->{'qdev_id'};
3378                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3379                     $devices->{$device->{'qdev_id'}}++;
3380                 }
3381             } else {
3382                 $devices->{$device->{'qdev_id'}} = 1;
3383             }
3384         }
3385     }
3386
3387     my $resblock = vm_mon_cmd($vmid, 'query-block');
3388     foreach my $block (@$resblock) {
3389         if($block->{device} =~ m/^drive-(\S+)/){
3390                 $devices->{$1} = 1;
3391         }
3392     }
3393
3394     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3395     foreach my $mice (@$resmice) {
3396         if ($mice->{name} eq 'QEMU HID Tablet') {
3397             $devices->{tablet} = 1;
3398             last;
3399         }
3400     }
3401
3402     # for usb devices there is no query-usb
3403     # but we can iterate over the entries in
3404     # qom-list path=/machine/peripheral
3405     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3406     foreach my $per (@$resperipheral) {
3407         if ($per->{name} =~ m/^usb\d+$/) {
3408             $devices->{$per->{name}} = 1;
3409         }
3410     }
3411
3412     return $devices;
3413 }
3414
3415 sub vm_deviceplug {
3416     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3417
3418     my $q35 = machine_type_is_q35($conf);
3419
3420     my $devices_list = vm_devices_list($vmid);
3421     return 1 if defined($devices_list->{$deviceid});
3422
3423     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3424
3425     if ($deviceid eq 'tablet') {
3426
3427         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3428
3429     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3430
3431         die "usb hotplug currently not reliable\n";
3432         # since we can't reliably hot unplug all added usb devices
3433         # and usb passthrough disables live migration
3434         # we disable usb hotplugging for now
3435         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3436
3437     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3438
3439         qemu_iothread_add($vmid, $deviceid, $device);
3440
3441         qemu_driveadd($storecfg, $vmid, $device);
3442         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3443
3444         qemu_deviceadd($vmid, $devicefull);
3445         eval { qemu_deviceaddverify($vmid, $deviceid); };
3446         if (my $err = $@) {
3447             eval { qemu_drivedel($vmid, $deviceid); };
3448             warn $@ if $@;
3449             die $err;
3450         }
3451
3452     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3453
3454
3455         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3456         my $pciaddr = print_pci_addr($deviceid);
3457         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3458
3459         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3460
3461         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3462             qemu_iothread_add($vmid, $deviceid, $device);
3463             $devicefull .= ",iothread=iothread-$deviceid";
3464         }
3465
3466         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3467             $devicefull .= ",num_queues=$device->{queues}";
3468         }
3469
3470         qemu_deviceadd($vmid, $devicefull);
3471         qemu_deviceaddverify($vmid, $deviceid);
3472
3473     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3474
3475         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3476         qemu_driveadd($storecfg, $vmid, $device);
3477
3478         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3479         eval { qemu_deviceadd($vmid, $devicefull); };
3480         if (my $err = $@) {
3481             eval { qemu_drivedel($vmid, $deviceid); };
3482             warn $@ if $@;
3483             die $err;
3484         }
3485
3486     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3487
3488         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3489
3490         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3491         my $use_old_bios_files = undef;
3492         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3493
3494         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3495         qemu_deviceadd($vmid, $netdevicefull);
3496         eval { qemu_deviceaddverify($vmid, $deviceid); };
3497         if (my $err = $@) {
3498             eval { qemu_netdevdel($vmid, $deviceid); };
3499             warn $@ if $@;
3500             die $err;
3501         }
3502
3503     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3504
3505         my $bridgeid = $2;
3506         my $pciaddr = print_pci_addr($deviceid);
3507         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3508
3509         qemu_deviceadd($vmid, $devicefull);
3510         qemu_deviceaddverify($vmid, $deviceid);
3511
3512     } else {
3513         die "can't hotplug device '$deviceid'\n";
3514     }
3515
3516     return 1;
3517 }
3518
3519 # fixme: this should raise exceptions on error!
3520 sub vm_deviceunplug {
3521     my ($vmid, $conf, $deviceid) = @_;
3522
3523     my $devices_list = vm_devices_list($vmid);
3524     return 1 if !defined($devices_list->{$deviceid});
3525
3526     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3527
3528     if ($deviceid eq 'tablet') {
3529
3530         qemu_devicedel($vmid, $deviceid);
3531
3532     } elsif ($deviceid =~ m/^usb\d+$/) {
3533
3534         die "usb hotplug currently not reliable\n";
3535         # when unplugging usb devices this way,
3536         # there may be remaining usb controllers/hubs
3537         # so we disable it for now
3538         qemu_devicedel($vmid, $deviceid);
3539         qemu_devicedelverify($vmid, $deviceid);
3540
3541     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3542
3543         qemu_devicedel($vmid, $deviceid);
3544         qemu_devicedelverify($vmid, $deviceid);
3545         qemu_drivedel($vmid, $deviceid);
3546         qemu_iothread_del($conf, $vmid, $deviceid);
3547
3548     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3549
3550         qemu_devicedel($vmid, $deviceid);
3551         qemu_devicedelverify($vmid, $deviceid);
3552         qemu_iothread_del($conf, $vmid, $deviceid);
3553
3554     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3555
3556         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3557         my $device = parse_drive($deviceid, $conf->{$deviceid});
3558         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3559
3560         qemu_devicedel($vmid, $deviceid);
3561         qemu_drivedel($vmid, $deviceid);
3562         qemu_deletescsihw($conf, $vmid, $deviceid);
3563
3564     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3565
3566         qemu_devicedel($vmid, $deviceid);
3567         qemu_devicedelverify($vmid, $deviceid);
3568         qemu_netdevdel($vmid, $deviceid);
3569
3570     } else {
3571         die "can't unplug device '$deviceid'\n";
3572     }
3573
3574     return 1;
3575 }
3576
3577 sub qemu_deviceadd {
3578     my ($vmid, $devicefull) = @_;
3579
3580     $devicefull = "driver=".$devicefull;
3581     my %options =  split(/[=,]/, $devicefull);
3582
3583     vm_mon_cmd($vmid, "device_add" , %options);
3584 }
3585
3586 sub qemu_devicedel {
3587     my ($vmid, $deviceid) = @_;
3588
3589     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3590 }
3591
3592 sub qemu_iothread_add {
3593     my($vmid, $deviceid, $device) = @_;
3594
3595     if ($device->{iothread}) {
3596         my $iothreads = vm_iothreads_list($vmid);
3597         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3598     }
3599 }
3600
3601 sub qemu_iothread_del {
3602     my($conf, $vmid, $deviceid) = @_;
3603
3604     my $device = parse_drive($deviceid, $conf->{$deviceid});
3605     if ($device->{iothread}) {
3606         my $iothreads = vm_iothreads_list($vmid);
3607         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3608     }
3609 }
3610
3611 sub qemu_objectadd {
3612     my($vmid, $objectid, $qomtype) = @_;
3613
3614     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3615
3616     return 1;
3617 }
3618
3619 sub qemu_objectdel {
3620     my($vmid, $objectid) = @_;
3621
3622     vm_mon_cmd($vmid, "object-del", id => $objectid);
3623
3624     return 1;
3625 }
3626
3627 sub qemu_driveadd {
3628     my ($storecfg, $vmid, $device) = @_;
3629
3630     my $drive = print_drive_full($storecfg, $vmid, $device);
3631     $drive =~ s/\\/\\\\/g;
3632     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3633
3634     # If the command succeeds qemu prints: "OK"
3635     return 1 if $ret =~ m/OK/s;
3636
3637     die "adding drive failed: $ret\n";
3638 }
3639
3640 sub qemu_drivedel {
3641     my($vmid, $deviceid) = @_;
3642
3643     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3644     $ret =~ s/^\s+//;
3645
3646     return 1 if $ret eq "";
3647
3648     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3649     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3650
3651     die "deleting drive $deviceid failed : $ret\n";
3652 }
3653
3654 sub qemu_deviceaddverify {
3655     my ($vmid, $deviceid) = @_;
3656
3657     for (my $i = 0; $i <= 5; $i++) {
3658          my $devices_list = vm_devices_list($vmid);
3659          return 1 if defined($devices_list->{$deviceid});
3660          sleep 1;
3661     }
3662
3663     die "error on hotplug device '$deviceid'\n";
3664 }
3665
3666
3667 sub qemu_devicedelverify {
3668     my ($vmid, $deviceid) = @_;
3669
3670     # need to verify that the device is correctly removed as device_del
3671     # is async and empty return is not reliable
3672
3673     for (my $i = 0; $i <= 5; $i++) {
3674          my $devices_list = vm_devices_list($vmid);
3675          return 1 if !defined($devices_list->{$deviceid});
3676          sleep 1;
3677     }
3678
3679     die "error on hot-unplugging device '$deviceid'\n";
3680 }
3681
3682 sub qemu_findorcreatescsihw {
3683     my ($storecfg, $conf, $vmid, $device) = @_;
3684
3685     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3686
3687     my $scsihwid="$controller_prefix$controller";
3688     my $devices_list = vm_devices_list($vmid);
3689
3690     if(!defined($devices_list->{$scsihwid})) {
3691         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3692     }
3693
3694     return 1;
3695 }
3696
3697 sub qemu_deletescsihw {
3698     my ($conf, $vmid, $opt) = @_;
3699
3700     my $device = parse_drive($opt, $conf->{$opt});
3701
3702     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3703         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3704         return 1;
3705     }
3706
3707     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3708
3709     my $devices_list = vm_devices_list($vmid);
3710     foreach my $opt (keys %{$devices_list}) {
3711         if (PVE::QemuServer::is_valid_drivename($opt)) {
3712             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3713             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3714                 return 1;
3715             }
3716         }
3717     }
3718
3719     my $scsihwid="scsihw$controller";
3720
3721     vm_deviceunplug($vmid, $conf, $scsihwid);
3722
3723     return 1;
3724 }
3725
3726 sub qemu_add_pci_bridge {
3727     my ($storecfg, $conf, $vmid, $device) = @_;
3728
3729     my $bridges = {};
3730
3731     my $bridgeid;
3732
3733     print_pci_addr($device, $bridges);
3734
3735     while (my ($k, $v) = each %$bridges) {
3736         $bridgeid = $k;
3737     }
3738     return 1 if !defined($bridgeid) || $bridgeid < 1;
3739
3740     my $bridge = "pci.$bridgeid";
3741     my $devices_list = vm_devices_list($vmid);
3742
3743     if (!defined($devices_list->{$bridge})) {
3744         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3745     }
3746
3747     return 1;
3748 }
3749
3750 sub qemu_set_link_status {
3751     my ($vmid, $device, $up) = @_;
3752
3753     vm_mon_cmd($vmid, "set_link", name => $device,
3754                up => $up ? JSON::true : JSON::false);
3755 }
3756
3757 sub qemu_netdevadd {
3758     my ($vmid, $conf, $device, $deviceid) = @_;
3759
3760     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3761     my %options =  split(/[=,]/, $netdev);
3762
3763     vm_mon_cmd($vmid, "netdev_add",  %options);
3764     return 1;
3765 }
3766
3767 sub qemu_netdevdel {
3768     my ($vmid, $deviceid) = @_;
3769
3770     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3771 }
3772
3773 sub qemu_usb_hotplug {
3774     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3775
3776     return if !$device;
3777
3778     # remove the old one first
3779     vm_deviceunplug($vmid, $conf, $deviceid);
3780
3781     # check if xhci controller is necessary and available
3782     if ($device->{usb3}) {
3783
3784         my $devicelist = vm_devices_list($vmid);
3785
3786         if (!$devicelist->{xhci}) {
3787             my $pciaddr = print_pci_addr("xhci");
3788             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3789         }
3790     }
3791     my $d = parse_usb_device($device->{host});
3792     $d->{usb3} = $device->{usb3};
3793
3794     # add the new one
3795     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3796 }
3797
3798 sub qemu_cpu_hotplug {
3799     my ($vmid, $conf, $vcpus) = @_;
3800
3801     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3802
3803     my $sockets = 1;
3804     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3805     $sockets = $conf->{sockets} if  $conf->{sockets};
3806     my $cores = $conf->{cores} || 1;
3807     my $maxcpus = $sockets * $cores;
3808
3809     $vcpus = $maxcpus if !$vcpus;
3810
3811     die "you can't add more vcpus than maxcpus\n"
3812         if $vcpus > $maxcpus;
3813
3814     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3815
3816     if ($vcpus < $currentvcpus) {
3817
3818         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3819
3820             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3821                 qemu_devicedel($vmid, "cpu$i");
3822                 my $retry = 0;
3823                 my $currentrunningvcpus = undef;
3824                 while (1) {
3825                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3826                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3827                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3828                     $retry++;
3829                     sleep 1;
3830                 }
3831                 #update conf after each succesfull cpu unplug
3832                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3833                 PVE::QemuConfig->write_config($vmid, $conf);
3834             }
3835         } else {
3836             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3837         }
3838
3839         return;
3840     }
3841
3842     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3843     die "vcpus in running vm does not match its configuration\n"
3844         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3845
3846     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3847
3848         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3849             my $cpustr = print_cpu_device($conf, $i);
3850             qemu_deviceadd($vmid, $cpustr);
3851
3852             my $retry = 0;
3853             my $currentrunningvcpus = undef;
3854             while (1) {
3855                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3856                 last if scalar(@{$currentrunningvcpus}) == $i;
3857                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3858                 sleep 1;
3859                 $retry++;
3860             }
3861             #update conf after each succesfull cpu hotplug
3862             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3863             PVE::QemuConfig->write_config($vmid, $conf);
3864         }
3865     } else {
3866
3867         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3868             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3869         }
3870     }
3871 }
3872
3873 sub qemu_block_set_io_throttle {
3874     my ($vmid, $deviceid,
3875         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3876         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3877         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3878         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3879
3880     return if !check_running($vmid) ;
3881
3882     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3883         bps => int($bps),
3884         bps_rd => int($bps_rd),
3885         bps_wr => int($bps_wr),
3886         iops => int($iops),
3887         iops_rd => int($iops_rd),
3888         iops_wr => int($iops_wr),
3889         bps_max => int($bps_max),
3890         bps_rd_max => int($bps_rd_max),
3891         bps_wr_max => int($bps_wr_max),
3892         iops_max => int($iops_max),
3893         iops_rd_max => int($iops_rd_max),
3894         iops_wr_max => int($iops_wr_max),
3895         bps_max_length => int($bps_max_length),
3896         bps_rd_max_length => int($bps_rd_max_length),
3897         bps_wr_max_length => int($bps_wr_max_length),
3898         iops_max_length => int($iops_max_length),
3899         iops_rd_max_length => int($iops_rd_max_length),
3900         iops_wr_max_length => int($iops_wr_max_length),
3901     );
3902
3903 }
3904
3905 # old code, only used to shutdown old VM after update
3906 sub __read_avail {
3907     my ($fh, $timeout) = @_;
3908
3909     my $sel = new IO::Select;
3910     $sel->add($fh);
3911
3912     my $res = '';
3913     my $buf;
3914
3915     my @ready;
3916     while (scalar (@ready = $sel->can_read($timeout))) {
3917         my $count;
3918         if ($count = $fh->sysread($buf, 8192)) {
3919             if ($buf =~ /^(.*)\(qemu\) $/s) {
3920                 $res .= $1;
3921                 last;
3922             } else {
3923                 $res .= $buf;
3924             }
3925         } else {
3926             if (!defined($count)) {
3927                 die "$!\n";
3928             }
3929             last;
3930         }
3931     }
3932
3933     die "monitor read timeout\n" if !scalar(@ready);
3934
3935     return $res;
3936 }
3937
3938 # old code, only used to shutdown old VM after update
3939 sub vm_monitor_command {
3940     my ($vmid, $cmdstr, $nocheck) = @_;
3941
3942     my $res;
3943
3944     eval {
3945         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3946
3947         my $sname = "${var_run_tmpdir}/$vmid.mon";
3948
3949         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3950             die "unable to connect to VM $vmid socket - $!\n";
3951
3952         my $timeout = 3;
3953
3954         # hack: migrate sometime blocks the monitor (when migrate_downtime
3955         # is set)
3956         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3957             $timeout = 60*60; # 1 hour
3958         }
3959
3960         # read banner;
3961         my $data = __read_avail($sock, $timeout);
3962
3963         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3964             die "got unexpected qemu monitor banner\n";
3965         }
3966
3967         my $sel = new IO::Select;
3968         $sel->add($sock);
3969
3970         if (!scalar(my @ready = $sel->can_write($timeout))) {
3971             die "monitor write error - timeout";
3972         }
3973
3974         my $fullcmd = "$cmdstr\r";
3975
3976         # syslog('info', "VM $vmid monitor command: $cmdstr");
3977
3978         my $b;
3979         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3980             die "monitor write error - $!";
3981         }
3982
3983         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3984
3985         $timeout = 20;
3986
3987         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3988             $timeout = 60*60; # 1 hour
3989         } elsif ($cmdstr =~ m/^(eject|change)/) {
3990             $timeout = 60; # note: cdrom mount command is slow
3991         }
3992         if ($res = __read_avail($sock, $timeout)) {
3993
3994             my @lines = split("\r?\n", $res);
3995
3996             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3997
3998             $res = join("\n", @lines);
3999             $res .= "\n";
4000         }
4001     };
4002
4003     my $err = $@;
4004
4005     if ($err) {
4006         syslog("err", "VM $vmid monitor command failed - $err");
4007         die $err;
4008     }
4009
4010     return $res;
4011 }
4012
4013 sub qemu_block_resize {
4014     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4015
4016     my $running = check_running($vmid);
4017
4018     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4019
4020     return if !$running;
4021
4022     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4023
4024 }
4025
4026 sub qemu_volume_snapshot {
4027     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4028
4029     my $running = check_running($vmid);
4030
4031     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4032         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4033     } else {
4034         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4035     }
4036 }
4037
4038 sub qemu_volume_snapshot_delete {
4039     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4040
4041     my $running = check_running($vmid);
4042
4043     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4044         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4045     } else {
4046         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4047     }
4048 }
4049
4050 sub set_migration_caps {
4051     my ($vmid) = @_;
4052
4053     my $cap_ref = [];
4054
4055     my $enabled_cap = {
4056         "auto-converge" => 1,
4057         "xbzrle" => 1,
4058         "x-rdma-pin-all" => 0,
4059         "zero-blocks" => 0,
4060         "compress" => 0
4061     };
4062
4063     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4064
4065     for my $supported_capability (@$supported_capabilities) {
4066         push @$cap_ref, {
4067             capability => $supported_capability->{capability},
4068             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4069         };
4070     }
4071
4072     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4073 }
4074
4075 my $fast_plug_option = {
4076     'lock' => 1,
4077     'name' => 1,
4078     'onboot' => 1,
4079     'shares' => 1,
4080     'startup' => 1,
4081     'description' => 1,
4082     'protection' => 1,
4083 };
4084
4085 # hotplug changes in [PENDING]
4086 # $selection hash can be used to only apply specified options, for
4087 # example: { cores => 1 } (only apply changed 'cores')
4088 # $errors ref is used to return error messages
4089 sub vmconfig_hotplug_pending {
4090     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4091
4092     my $defaults = load_defaults();
4093
4094     # commit values which do not have any impact on running VM first
4095     # Note: those option cannot raise errors, we we do not care about
4096     # $selection and always apply them.
4097
4098     my $add_error = sub {
4099         my ($opt, $msg) = @_;
4100         $errors->{$opt} = "hotplug problem - $msg";
4101     };
4102
4103     my $changes = 0;
4104     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4105         if ($fast_plug_option->{$opt}) {
4106             $conf->{$opt} = $conf->{pending}->{$opt};
4107             delete $conf->{pending}->{$opt};
4108             $changes = 1;
4109         }
4110     }
4111
4112     if ($changes) {
4113         PVE::QemuConfig->write_config($vmid, $conf);
4114         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4115     }
4116
4117     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4118
4119     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4120     while (my ($opt, $force) = each %$pending_delete_hash) {
4121         next if $selection && !$selection->{$opt};
4122         eval {
4123             if ($opt eq 'hotplug') {
4124                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4125             } elsif ($opt eq 'tablet') {
4126                 die "skip\n" if !$hotplug_features->{usb};
4127                 if ($defaults->{tablet}) {
4128                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4129                 } else {
4130                     vm_deviceunplug($vmid, $conf, $opt);
4131                 }
4132             } elsif ($opt =~ m/^usb\d+/) {
4133                 die "skip\n";
4134                 # since we cannot reliably hot unplug usb devices
4135                 # we are disabling it
4136                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4137                 vm_deviceunplug($vmid, $conf, $opt);
4138             } elsif ($opt eq 'vcpus') {
4139                 die "skip\n" if !$hotplug_features->{cpu};
4140                 qemu_cpu_hotplug($vmid, $conf, undef);
4141             } elsif ($opt eq 'balloon') {
4142                 # enable balloon device is not hotpluggable
4143                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4144             } elsif ($fast_plug_option->{$opt}) {
4145                 # do nothing
4146             } elsif ($opt =~ m/^net(\d+)$/) {
4147                 die "skip\n" if !$hotplug_features->{network};
4148                 vm_deviceunplug($vmid, $conf, $opt);
4149             } elsif (is_valid_drivename($opt)) {
4150                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4151                 vm_deviceunplug($vmid, $conf, $opt);
4152                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4153             } elsif ($opt =~ m/^memory$/) {
4154                 die "skip\n" if !$hotplug_features->{memory};
4155                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4156             } elsif ($opt eq 'cpuunits') {
4157                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4158             } elsif ($opt eq 'cpulimit') {
4159                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4160             } else {
4161                 die "skip\n";
4162             }
4163         };
4164         if (my $err = $@) {
4165             &$add_error($opt, $err) if $err ne "skip\n";
4166         } else {
4167             # save new config if hotplug was successful
4168             delete $conf->{$opt};
4169             vmconfig_undelete_pending_option($conf, $opt);
4170             PVE::QemuConfig->write_config($vmid, $conf);
4171             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4172         }
4173     }
4174
4175     foreach my $opt (keys %{$conf->{pending}}) {
4176         next if $selection && !$selection->{$opt};
4177         my $value = $conf->{pending}->{$opt};
4178         eval {
4179             if ($opt eq 'hotplug') {
4180                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4181             } elsif ($opt eq 'tablet') {
4182                 die "skip\n" if !$hotplug_features->{usb};
4183                 if ($value == 1) {
4184                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4185                 } elsif ($value == 0) {
4186                     vm_deviceunplug($vmid, $conf, $opt);
4187                 }
4188             } elsif ($opt =~ m/^usb\d+$/) {
4189                 die "skip\n";
4190                 # since we cannot reliably hot unplug usb devices
4191                 # we are disabling it
4192                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4193                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4194                 die "skip\n" if !$d;
4195                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4196             } elsif ($opt eq 'vcpus') {
4197                 die "skip\n" if !$hotplug_features->{cpu};
4198                 qemu_cpu_hotplug($vmid, $conf, $value);
4199             } elsif ($opt eq 'balloon') {
4200                 # enable/disable balloning device is not hotpluggable
4201                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4202                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4203                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4204
4205                 # allow manual ballooning if shares is set to zero
4206                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4207                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4208                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4209                 }
4210             } elsif ($opt =~ m/^net(\d+)$/) {
4211                 # some changes can be done without hotplug
4212                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4213                                     $vmid, $opt, $value);
4214             } elsif (is_valid_drivename($opt)) {
4215                 # some changes can be done without hotplug
4216                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4217                                      $vmid, $opt, $value, 1);
4218             } elsif ($opt =~ m/^memory$/) { #dimms
4219                 die "skip\n" if !$hotplug_features->{memory};
4220                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4221             } elsif ($opt eq 'cpuunits') {
4222                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4223             } elsif ($opt eq 'cpulimit') {
4224                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4225                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4226             } else {
4227                 die "skip\n";  # skip non-hot-pluggable options
4228             }
4229         };
4230         if (my $err = $@) {
4231             &$add_error($opt, $err) if $err ne "skip\n";
4232         } else {
4233             # save new config if hotplug was successful
4234             $conf->{$opt} = $value;
4235             delete $conf->{pending}->{$opt};
4236             PVE::QemuConfig->write_config($vmid, $conf);
4237             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4238         }
4239     }
4240 }
4241
4242 sub try_deallocate_drive {
4243     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4244
4245     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4246         my $volid = $drive->{file};
4247         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4248             my $sid = PVE::Storage::parse_volume_id($volid);
4249             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4250
4251             # check if the disk is really unused
4252             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4253                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4254             PVE::Storage::vdisk_free($storecfg, $volid);
4255             return 1;
4256         } else {
4257             # If vm is not owner of this disk remove from config
4258             return 1;
4259         }
4260     }
4261
4262     return undef;
4263 }
4264
4265 sub vmconfig_delete_or_detach_drive {
4266     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4267
4268     my $drive = parse_drive($opt, $conf->{$opt});
4269
4270     my $rpcenv = PVE::RPCEnvironment::get();
4271     my $authuser = $rpcenv->get_user();
4272
4273     if ($force) {
4274         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4275         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4276     } else {
4277         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4278     }
4279 }
4280
4281 sub vmconfig_apply_pending {
4282     my ($vmid, $conf, $storecfg) = @_;
4283
4284     # cold plug
4285
4286     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4287     while (my ($opt, $force) = each %$pending_delete_hash) {
4288         die "internal error" if $opt =~ m/^unused/;
4289         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4290         if (!defined($conf->{$opt})) {
4291             vmconfig_undelete_pending_option($conf, $opt);
4292             PVE::QemuConfig->write_config($vmid, $conf);
4293         } elsif (is_valid_drivename($opt)) {
4294             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4295             vmconfig_undelete_pending_option($conf, $opt);
4296             delete $conf->{$opt};
4297             PVE::QemuConfig->write_config($vmid, $conf);
4298         } else {
4299             vmconfig_undelete_pending_option($conf, $opt);
4300             delete $conf->{$opt};
4301             PVE::QemuConfig->write_config($vmid, $conf);
4302         }
4303     }
4304
4305     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4306
4307     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4308         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4309
4310         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4311             # skip if nothing changed
4312         } elsif (is_valid_drivename($opt)) {
4313             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4314                 if defined($conf->{$opt});
4315             $conf->{$opt} = $conf->{pending}->{$opt};
4316         } else {
4317             $conf->{$opt} = $conf->{pending}->{$opt};
4318         }
4319
4320         delete $conf->{pending}->{$opt};
4321         PVE::QemuConfig->write_config($vmid, $conf);
4322     }
4323 }
4324
4325 my $safe_num_ne = sub {
4326     my ($a, $b) = @_;
4327
4328     return 0 if !defined($a) && !defined($b);
4329     return 1 if !defined($a);
4330     return 1 if !defined($b);
4331
4332     return $a != $b;
4333 };
4334
4335 my $safe_string_ne = sub {
4336     my ($a, $b) = @_;
4337
4338     return 0 if !defined($a) && !defined($b);
4339     return 1 if !defined($a);
4340     return 1 if !defined($b);
4341
4342     return $a ne $b;
4343 };
4344
4345 sub vmconfig_update_net {
4346     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4347
4348     my $newnet = parse_net($value);
4349
4350     if ($conf->{$opt}) {
4351         my $oldnet = parse_net($conf->{$opt});
4352
4353         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4354             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4355             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4356             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4357
4358             # for non online change, we try to hot-unplug
4359             die "skip\n" if !$hotplug;
4360             vm_deviceunplug($vmid, $conf, $opt);
4361         } else {
4362
4363             die "internal error" if $opt !~ m/net(\d+)/;
4364             my $iface = "tap${vmid}i$1";
4365
4366             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4367                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4368                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4369                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4370                 PVE::Network::tap_unplug($iface);
4371                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4372             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4373                 # Rate can be applied on its own but any change above needs to
4374                 # include the rate in tap_plug since OVS resets everything.
4375                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4376             }
4377
4378             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4379                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4380             }
4381
4382             return 1;
4383         }
4384     }
4385
4386     if ($hotplug) {
4387         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4388     } else {
4389         die "skip\n";
4390     }
4391 }
4392
4393 sub vmconfig_update_disk {
4394     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4395
4396     # fixme: do we need force?
4397
4398     my $drive = parse_drive($opt, $value);
4399
4400     if ($conf->{$opt}) {
4401
4402         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4403
4404             my $media = $drive->{media} || 'disk';
4405             my $oldmedia = $old_drive->{media} || 'disk';
4406             die "unable to change media type\n" if $media ne $oldmedia;
4407
4408             if (!drive_is_cdrom($old_drive)) {
4409
4410                 if ($drive->{file} ne $old_drive->{file}) {
4411
4412                     die "skip\n" if !$hotplug;
4413
4414                     # unplug and register as unused
4415                     vm_deviceunplug($vmid, $conf, $opt);
4416                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4417
4418                 } else {
4419                     # update existing disk
4420
4421                     # skip non hotpluggable value
4422                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4423                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4424                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4425                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4426                         die "skip\n";
4427                     }
4428
4429                     # apply throttle
4430                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4431                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4432                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4433                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4434                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4435                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4436                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4437                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4438                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4439                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4440                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4441                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4442                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4443                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4444                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4445                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4446                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4447                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4448
4449                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4450                                                    ($drive->{mbps} || 0)*1024*1024,
4451                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4452                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4453                                                    $drive->{iops} || 0,
4454                                                    $drive->{iops_rd} || 0,
4455                                                    $drive->{iops_wr} || 0,
4456                                                    ($drive->{mbps_max} || 0)*1024*1024,
4457                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4458                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4459                                                    $drive->{iops_max} || 0,
4460                                                    $drive->{iops_rd_max} || 0,
4461                                                    $drive->{iops_wr_max} || 0,
4462                                                    $drive->{bps_max_length} || 1,
4463                                                    $drive->{bps_rd_max_length} || 1,
4464                                                    $drive->{bps_wr_max_length} || 1,
4465                                                    $drive->{iops_max_length} || 1,
4466                                                    $drive->{iops_rd_max_length} || 1,
4467                                                    $drive->{iops_wr_max_length} || 1);
4468
4469                     }
4470
4471                     return 1;
4472                 }
4473
4474             } else { # cdrom
4475
4476                 if ($drive->{file} eq 'none') {
4477                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4478                 } else {
4479                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4480                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4481                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4482                 }
4483
4484                 return 1;
4485             }
4486         }
4487     }
4488
4489     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4490     # hotplug new disks
4491     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4492     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4493 }
4494
4495 sub vm_start {
4496     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4497         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4498
4499     PVE::QemuConfig->lock_config($vmid, sub {
4500         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4501
4502         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4503
4504         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4505
4506         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4507
4508         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4509             vmconfig_apply_pending($vmid, $conf, $storecfg);
4510             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4511         }
4512
4513         my $defaults = load_defaults();
4514
4515         # set environment variable useful inside network script
4516         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4517
4518         my $local_volumes = {};
4519
4520         if ($targetstorage) {
4521             foreach_drive($conf, sub {
4522                 my ($ds, $drive) = @_;
4523
4524                 return if drive_is_cdrom($drive);
4525
4526                 my $volid = $drive->{file};
4527
4528                 return if !$volid;
4529
4530                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4531
4532                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4533                 return if $scfg->{shared};
4534                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4535             });
4536
4537             my $format = undef;
4538
4539             foreach my $opt (sort keys %$local_volumes) {
4540
4541                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4542                 my $drive = parse_drive($opt, $conf->{$opt});
4543
4544                 #if remote storage is specified, use default format
4545                 if ($targetstorage && $targetstorage ne "1") {
4546                     $storeid = $targetstorage;
4547                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4548                     $format = $defFormat;
4549                 } else {
4550                     #else we use same format than original
4551                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4552                     $format = qemu_img_format($scfg, $volid);
4553                 }
4554
4555                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4556                 my $newdrive = $drive;
4557                 $newdrive->{format} = $format;
4558                 $newdrive->{file} = $newvolid;
4559                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4560                 $local_volumes->{$opt} = $drivestr;
4561                 #pass drive to conf for command line
4562                 $conf->{$opt} = $drivestr;
4563             }
4564         }
4565
4566         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4567
4568         my $migrate_port = 0;
4569         my $migrate_uri;
4570         if ($statefile) {
4571             if ($statefile eq 'tcp') {
4572                 my $localip = "localhost";
4573                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4574                 my $nodename = PVE::INotify::nodename();
4575
4576                 if (!defined($migration_type)) {
4577                     if (defined($datacenterconf->{migration}->{type})) {
4578                         $migration_type = $datacenterconf->{migration}->{type};
4579                     } else {
4580                         $migration_type = 'secure';
4581                     }
4582                 }
4583
4584                 if ($migration_type eq 'insecure') {
4585                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4586                     if ($migrate_network_addr) {
4587                         $localip = $migrate_network_addr;
4588                     } else {
4589                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4590                     }
4591
4592                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4593                 }
4594
4595                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4596                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4597                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4598                 push @$cmd, '-incoming', $migrate_uri;
4599                 push @$cmd, '-S';
4600
4601             } elsif ($statefile eq 'unix') {
4602                 # should be default for secure migrations as a ssh TCP forward
4603                 # tunnel is not deterministic reliable ready and fails regurarly
4604                 # to set up in time, so use UNIX socket forwards
4605                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4606                 unlink $socket_addr;
4607
4608                 $migrate_uri = "unix:$socket_addr";
4609
4610                 push @$cmd, '-incoming', $migrate_uri;
4611                 push @$cmd, '-S';
4612
4613             } else {
4614                 push @$cmd, '-loadstate', $statefile;
4615             }
4616         } elsif ($paused) {
4617             push @$cmd, '-S';
4618         }
4619
4620         # host pci devices
4621         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4622           my $d = parse_hostpci($conf->{"hostpci$i"});
4623           next if !$d;
4624           my $pcidevices = $d->{pciid};
4625           foreach my $pcidevice (@$pcidevices) {
4626                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4627
4628                 my $info = pci_device_info("0000:$pciid");
4629                 die "IOMMU not present\n" if !check_iommu_support();
4630                 die "no pci device info for device '$pciid'\n" if !$info;
4631                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4632                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4633           }
4634         }
4635
4636         PVE::Storage::activate_volumes($storecfg, $vollist);
4637
4638         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4639             my $cmd = [];
4640             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4641             eval  { run_command($cmd); };
4642         }
4643
4644         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4645                                                   : $defaults->{cpuunits};
4646
4647         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4648
4649         my %properties = (
4650             Slice => 'qemu.slice',
4651             KillMode => 'none',
4652             CPUShares => $cpuunits
4653         );
4654
4655         if (my $cpulimit = $conf->{cpulimit}) {
4656             $properties{CPUQuota} = int($cpulimit * 100);
4657         }
4658         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4659
4660         if ($conf->{hugepages}) {
4661
4662             my $code = sub {
4663                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4664                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4665
4666                 PVE::QemuServer::Memory::hugepages_mount();
4667                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4668
4669                 eval  {
4670                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4671                     run_command($cmd, %run_params);
4672                 };
4673
4674                 if (my $err = $@) {
4675                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4676                     die $err;
4677                 }
4678
4679                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4680             };
4681             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4682
4683         } else {
4684             eval  {
4685                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4686                 run_command($cmd, %run_params);
4687             };
4688         }
4689
4690         if (my $err = $@) {
4691             # deactivate volumes if start fails
4692             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4693             die "start failed: $err";
4694         }
4695
4696         print "migration listens on $migrate_uri\n" if $migrate_uri;
4697
4698         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4699             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4700             warn $@ if $@;
4701         }
4702
4703         #start nbd server for storage migration
4704         if ($targetstorage) {
4705             my $nodename = PVE::INotify::nodename();
4706             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4707             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4708             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4709             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4710
4711             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4712
4713             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4714
4715             foreach my $opt (sort keys %$local_volumes) {
4716                 my $volid = $local_volumes->{$opt};
4717                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4718                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4719                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4720             }
4721         }
4722
4723         if ($migratedfrom) {
4724             eval {
4725                 set_migration_caps($vmid);
4726             };
4727             warn $@ if $@;
4728
4729             if ($spice_port) {
4730                 print "spice listens on port $spice_port\n";
4731                 if ($spice_ticket) {
4732                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4733                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4734                 }
4735             }
4736
4737         } else {
4738             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4739                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4740                     if $conf->{balloon};
4741             }
4742
4743             foreach my $opt (keys %$conf) {
4744                 next if $opt !~  m/^net\d+$/;
4745                 my $nicconf = parse_net($conf->{$opt});
4746                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4747             }
4748         }
4749
4750         vm_mon_cmd_nocheck($vmid, 'qom-set',
4751                     path => "machine/peripheral/balloon0",
4752                     property => "guest-stats-polling-interval",
4753                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4754
4755     });
4756 }
4757
4758 sub vm_mon_cmd {
4759     my ($vmid, $execute, %params) = @_;
4760
4761     my $cmd = { execute => $execute, arguments => \%params };
4762     vm_qmp_command($vmid, $cmd);
4763 }
4764
4765 sub vm_mon_cmd_nocheck {
4766     my ($vmid, $execute, %params) = @_;
4767
4768     my $cmd = { execute => $execute, arguments => \%params };
4769     vm_qmp_command($vmid, $cmd, 1);
4770 }
4771
4772 sub vm_qmp_command {
4773     my ($vmid, $cmd, $nocheck) = @_;
4774
4775     my $res;
4776
4777     my $timeout;
4778     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4779         $timeout = $cmd->{arguments}->{timeout};
4780         delete $cmd->{arguments}->{timeout};
4781     }
4782
4783     eval {
4784         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4785         my $sname = qmp_socket($vmid);
4786         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4787             my $qmpclient = PVE::QMPClient->new();
4788
4789             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4790         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4791             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4792                 if scalar(%{$cmd->{arguments}});
4793             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4794         } else {
4795             die "unable to open monitor socket\n";
4796         }
4797     };
4798     if (my $err = $@) {
4799         syslog("err", "VM $vmid qmp command failed - $err");
4800         die $err;
4801     }
4802
4803     return $res;
4804 }
4805
4806 sub vm_human_monitor_command {
4807     my ($vmid, $cmdline) = @_;
4808
4809     my $res;
4810
4811     my $cmd = {
4812         execute => 'human-monitor-command',
4813         arguments => { 'command-line' => $cmdline},
4814     };
4815
4816     return vm_qmp_command($vmid, $cmd);
4817 }
4818
4819 sub vm_commandline {
4820     my ($storecfg, $vmid) = @_;
4821
4822     my $conf = PVE::QemuConfig->load_config($vmid);
4823
4824     my $defaults = load_defaults();
4825
4826     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4827
4828     return PVE::Tools::cmd2string($cmd);
4829 }
4830
4831 sub vm_reset {
4832     my ($vmid, $skiplock) = @_;
4833
4834     PVE::QemuConfig->lock_config($vmid, sub {
4835
4836         my $conf = PVE::QemuConfig->load_config($vmid);
4837
4838         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4839
4840         vm_mon_cmd($vmid, "system_reset");
4841     });
4842 }
4843
4844 sub get_vm_volumes {
4845     my ($conf) = @_;
4846
4847     my $vollist = [];
4848     foreach_volid($conf, sub {
4849         my ($volid, $is_cdrom) = @_;
4850
4851         return if $volid =~ m|^/|;
4852
4853         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4854         return if !$sid;
4855
4856         push @$vollist, $volid;
4857     });
4858
4859     return $vollist;
4860 }
4861
4862 sub vm_stop_cleanup {
4863     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4864
4865     eval {
4866
4867         if (!$keepActive) {
4868             my $vollist = get_vm_volumes($conf);
4869             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4870         }
4871
4872         foreach my $ext (qw(mon qmp pid vnc qga)) {
4873             unlink "/var/run/qemu-server/${vmid}.$ext";
4874         }
4875
4876         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4877     };
4878     warn $@ if $@; # avoid errors - just warn
4879 }
4880
4881 # Note: use $nockeck to skip tests if VM configuration file exists.
4882 # We need that when migration VMs to other nodes (files already moved)
4883 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4884 sub vm_stop {
4885     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4886
4887     $force = 1 if !defined($force) && !$shutdown;
4888
4889     if ($migratedfrom){
4890         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4891         kill 15, $pid if $pid;
4892         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4893         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4894         return;
4895     }
4896
4897     PVE::QemuConfig->lock_config($vmid, sub {
4898
4899         my $pid = check_running($vmid, $nocheck);
4900         return if !$pid;
4901
4902         my $conf;
4903         if (!$nocheck) {
4904             $conf = PVE::QemuConfig->load_config($vmid);
4905             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4906             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4907                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4908                 $timeout = $opts->{down} if $opts->{down};
4909             }
4910         }
4911
4912         $timeout = 60 if !defined($timeout);
4913
4914         eval {
4915             if ($shutdown) {
4916                 if (defined($conf) && $conf->{agent}) {
4917                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4918                 } else {
4919                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4920                 }
4921             } else {
4922                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4923             }
4924         };
4925         my $err = $@;
4926
4927         if (!$err) {
4928             my $count = 0;
4929             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4930                 $count++;
4931                 sleep 1;
4932             }
4933
4934             if ($count >= $timeout) {
4935                 if ($force) {
4936                     warn "VM still running - terminating now with SIGTERM\n";
4937                     kill 15, $pid;
4938                 } else {
4939                     die "VM quit/powerdown failed - got timeout\n";
4940                 }
4941             } else {
4942                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4943                 return;
4944             }
4945         } else {
4946             if ($force) {
4947                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4948                 kill 15, $pid;
4949             } else {
4950                 die "VM quit/powerdown failed\n";
4951             }
4952         }
4953
4954         # wait again
4955         $timeout = 10;
4956
4957         my $count = 0;
4958         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4959             $count++;
4960             sleep 1;
4961         }
4962
4963         if ($count >= $timeout) {
4964             warn "VM still running - terminating now with SIGKILL\n";
4965             kill 9, $pid;
4966             sleep 1;
4967         }
4968
4969         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4970    });
4971 }
4972
4973 sub vm_suspend {
4974     my ($vmid, $skiplock) = @_;
4975
4976     PVE::QemuConfig->lock_config($vmid, sub {
4977
4978         my $conf = PVE::QemuConfig->load_config($vmid);
4979
4980         PVE::QemuConfig->check_lock($conf)
4981             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4982
4983         vm_mon_cmd($vmid, "stop");
4984     });
4985 }
4986
4987 sub vm_resume {
4988     my ($vmid, $skiplock, $nocheck) = @_;
4989
4990     PVE::QemuConfig->lock_config($vmid, sub {
4991
4992         if (!$nocheck) {
4993
4994             my $conf = PVE::QemuConfig->load_config($vmid);
4995
4996             PVE::QemuConfig->check_lock($conf)
4997                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4998
4999             vm_mon_cmd($vmid, "cont");
5000
5001         } else {
5002             vm_mon_cmd_nocheck($vmid, "cont");
5003         }
5004     });
5005 }
5006
5007 sub vm_sendkey {
5008     my ($vmid, $skiplock, $key) = @_;
5009
5010     PVE::QemuConfig->lock_config($vmid, sub {
5011
5012         my $conf = PVE::QemuConfig->load_config($vmid);
5013
5014         # there is no qmp command, so we use the human monitor command
5015         vm_human_monitor_command($vmid, "sendkey $key");
5016     });
5017 }
5018
5019 sub vm_destroy {
5020     my ($storecfg, $vmid, $skiplock) = @_;
5021
5022     PVE::QemuConfig->lock_config($vmid, sub {
5023
5024         my $conf = PVE::QemuConfig->load_config($vmid);
5025
5026         if (!check_running($vmid)) {
5027             destroy_vm($storecfg, $vmid, undef, $skiplock);
5028         } else {
5029             die "VM $vmid is running - destroy failed\n";
5030         }
5031     });
5032 }
5033
5034 # pci helpers
5035
5036 sub file_write {
5037     my ($filename, $buf) = @_;
5038
5039     my $fh = IO::File->new($filename, "w");
5040     return undef if !$fh;
5041
5042     my $res = print $fh $buf;
5043
5044     $fh->close();
5045
5046     return $res;
5047 }
5048
5049 sub pci_device_info {
5050     my ($name) = @_;
5051
5052     my $res;
5053
5054     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5055     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5056
5057     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5058     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5059
5060     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5061     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5062
5063     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5064     return undef if !defined($product) || $product !~ s/^0x//;
5065
5066     $res = {
5067         name => $name,
5068         vendor => $vendor,
5069         product => $product,
5070         domain => $domain,
5071         bus => $bus,
5072         slot => $slot,
5073         func => $func,
5074         irq => $irq,
5075         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5076     };
5077
5078     return $res;
5079 }
5080
5081 sub pci_dev_reset {
5082     my ($dev) = @_;
5083
5084     my $name = $dev->{name};
5085
5086     my $fn = "$pcisysfs/devices/$name/reset";
5087
5088     return file_write($fn, "1");
5089 }
5090
5091 sub pci_dev_bind_to_vfio {
5092     my ($dev) = @_;
5093
5094     my $name = $dev->{name};
5095
5096     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5097
5098     if (!-d $vfio_basedir) {
5099         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5100     }
5101     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5102
5103     my $testdir = "$vfio_basedir/$name";
5104     return 1 if -d $testdir;
5105
5106     my $data = "$dev->{vendor} $dev->{product}";
5107     return undef if !file_write("$vfio_basedir/new_id", $data);
5108
5109     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5110     if (!file_write($fn, $name)) {
5111         return undef if -f $fn;
5112     }
5113
5114     $fn = "$vfio_basedir/bind";
5115     if (! -d $testdir) {
5116         return undef if !file_write($fn, $name);
5117     }
5118
5119     return -d $testdir;
5120 }
5121
5122 sub pci_dev_group_bind_to_vfio {
5123     my ($pciid) = @_;
5124
5125     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5126
5127     if (!-d $vfio_basedir) {
5128         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5129     }
5130     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5131
5132     # get IOMMU group devices
5133     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5134       my @devs = grep /^0000:/, readdir($D);
5135     closedir($D);
5136
5137     foreach my $pciid (@devs) {
5138         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5139
5140         # pci bridges, switches or root ports are not supported
5141         # they have a pci_bus subdirectory so skip them
5142         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5143
5144         my $info = pci_device_info($1);
5145         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5146     }
5147
5148     return 1;
5149 }
5150
5151 # vzdump restore implementaion
5152
5153 sub tar_archive_read_firstfile {
5154     my $archive = shift;
5155
5156     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5157
5158     # try to detect archive type first
5159     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5160         die "unable to open file '$archive'\n";
5161     my $firstfile = <$fh>;
5162     kill 15, $pid;
5163     close $fh;
5164
5165     die "ERROR: archive contaions no data\n" if !$firstfile;
5166     chomp $firstfile;
5167
5168     return $firstfile;
5169 }
5170
5171 sub tar_restore_cleanup {
5172     my ($storecfg, $statfile) = @_;
5173
5174     print STDERR "starting cleanup\n";
5175
5176     if (my $fd = IO::File->new($statfile, "r")) {
5177         while (defined(my $line = <$fd>)) {
5178             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5179                 my $volid = $2;
5180                 eval {
5181                     if ($volid =~ m|^/|) {
5182                         unlink $volid || die 'unlink failed\n';
5183                     } else {
5184                         PVE::Storage::vdisk_free($storecfg, $volid);
5185                     }
5186                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5187                 };
5188                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5189             } else {
5190                 print STDERR "unable to parse line in statfile - $line";
5191             }
5192         }
5193         $fd->close();
5194     }
5195 }
5196
5197 sub restore_archive {
5198     my ($archive, $vmid, $user, $opts) = @_;
5199
5200     my $format = $opts->{format};
5201     my $comp;
5202
5203     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5204         $format = 'tar' if !$format;
5205         $comp = 'gzip';
5206     } elsif ($archive =~ m/\.tar$/) {
5207         $format = 'tar' if !$format;
5208     } elsif ($archive =~ m/.tar.lzo$/) {
5209         $format = 'tar' if !$format;
5210         $comp = 'lzop';
5211     } elsif ($archive =~ m/\.vma$/) {
5212         $format = 'vma' if !$format;
5213     } elsif ($archive =~ m/\.vma\.gz$/) {
5214         $format = 'vma' if !$format;
5215         $comp = 'gzip';
5216     } elsif ($archive =~ m/\.vma\.lzo$/) {
5217         $format = 'vma' if !$format;
5218         $comp = 'lzop';
5219     } else {
5220         $format = 'vma' if !$format; # default
5221     }
5222
5223     # try to detect archive format
5224     if ($format eq 'tar') {
5225         return restore_tar_archive($archive, $vmid, $user, $opts);
5226     } else {
5227         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5228     }
5229 }
5230
5231 sub restore_update_config_line {
5232     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5233
5234     return if $line =~ m/^\#qmdump\#/;
5235     return if $line =~ m/^\#vzdump\#/;
5236     return if $line =~ m/^lock:/;
5237     return if $line =~ m/^unused\d+:/;
5238     return if $line =~ m/^parent:/;
5239     return if $line =~ m/^template:/; # restored VM is never a template
5240
5241     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5242     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5243         # try to convert old 1.X settings
5244         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5245         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5246             my ($model, $macaddr) = split(/\=/, $devconfig);
5247             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5248             my $net = {
5249                 model => $model,
5250                 bridge => "vmbr$ind",
5251                 macaddr => $macaddr,
5252             };
5253             my $netstr = print_net($net);
5254
5255             print $outfd "net$cookie->{netcount}: $netstr\n";
5256             $cookie->{netcount}++;
5257         }
5258     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5259         my ($id, $netstr) = ($1, $2);
5260         my $net = parse_net($netstr);
5261         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5262         $netstr = print_net($net);
5263         print $outfd "$id: $netstr\n";
5264     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5265         my $virtdev = $1;
5266         my $value = $3;
5267         my $di = parse_drive($virtdev, $value);
5268         if (defined($di->{backup}) && !$di->{backup}) {
5269             print $outfd "#$line";
5270         } elsif ($map->{$virtdev}) {
5271             delete $di->{format}; # format can change on restore
5272             $di->{file} = $map->{$virtdev};
5273             $value = print_drive($vmid, $di);
5274             print $outfd "$virtdev: $value\n";
5275         } else {
5276             print $outfd $line;
5277         }
5278     } else {
5279         print $outfd $line;
5280     }
5281 }
5282
5283 sub scan_volids {
5284     my ($cfg, $vmid) = @_;
5285
5286     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5287
5288     my $volid_hash = {};
5289     foreach my $storeid (keys %$info) {
5290         foreach my $item (@{$info->{$storeid}}) {
5291             next if !($item->{volid} && $item->{size});
5292             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5293             $volid_hash->{$item->{volid}} = $item;
5294         }
5295     }
5296
5297     return $volid_hash;
5298 }
5299
5300 sub is_volume_in_use {
5301     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5302
5303     my $path = PVE::Storage::path($storecfg, $volid);
5304
5305     my $scan_config = sub {
5306         my ($cref, $snapname) = @_;
5307
5308         foreach my $key (keys %$cref) {
5309             my $value = $cref->{$key};
5310             if (is_valid_drivename($key)) {
5311                 next if $skip_drive && $key eq $skip_drive;
5312                 my $drive = parse_drive($key, $value);
5313                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5314                 return 1 if $volid eq $drive->{file};
5315                 if ($drive->{file} =~ m!^/!) {
5316                     return 1 if $drive->{file} eq $path;
5317                 } else {
5318                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5319                     next if !$storeid;
5320                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5321                     next if !$scfg;
5322                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5323                 }
5324             }
5325         }
5326
5327         return 0;
5328     };
5329
5330     return 1 if &$scan_config($conf);
5331
5332     undef $skip_drive;
5333
5334     foreach my $snapname (keys %{$conf->{snapshots}}) {
5335         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5336     }
5337
5338     return 0;
5339 }
5340
5341 sub update_disksize {
5342     my ($vmid, $conf, $volid_hash) = @_;
5343
5344     my $changes;
5345
5346     my $used = {};
5347
5348     # Note: it is allowed to define multiple storages with same path (alias), so
5349     # we need to check both 'volid' and real 'path' (two different volid can point
5350     # to the same path).
5351
5352     my $usedpath = {};
5353
5354     # update size info
5355     foreach my $opt (keys %$conf) {
5356         if (is_valid_drivename($opt)) {
5357             my $drive = parse_drive($opt, $conf->{$opt});
5358             my $volid = $drive->{file};
5359             next if !$volid;
5360
5361             $used->{$volid} = 1;
5362             if ($volid_hash->{$volid} &&
5363                 (my $path = $volid_hash->{$volid}->{path})) {
5364                 $usedpath->{$path} = 1;
5365             }
5366
5367             next if drive_is_cdrom($drive);
5368             next if !$volid_hash->{$volid};
5369
5370             $drive->{size} = $volid_hash->{$volid}->{size};
5371             my $new = print_drive($vmid, $drive);
5372             if ($new ne $conf->{$opt}) {
5373                 $changes = 1;
5374                 $conf->{$opt} = $new;
5375             }
5376         }
5377     }
5378
5379     # remove 'unusedX' entry if volume is used
5380     foreach my $opt (keys %$conf) {
5381         next if $opt !~ m/^unused\d+$/;
5382         my $volid = $conf->{$opt};
5383         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5384         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5385             $changes = 1;
5386             delete $conf->{$opt};
5387         }
5388     }
5389
5390     foreach my $volid (sort keys %$volid_hash) {
5391         next if $volid =~ m/vm-$vmid-state-/;
5392         next if $used->{$volid};
5393         my $path = $volid_hash->{$volid}->{path};
5394         next if !$path; # just to be sure
5395         next if $usedpath->{$path};
5396         $changes = 1;
5397         PVE::QemuConfig->add_unused_volume($conf, $volid);
5398         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5399     }
5400
5401     return $changes;
5402 }
5403
5404 sub rescan {
5405     my ($vmid, $nolock) = @_;
5406
5407     my $cfg = PVE::Storage::config();
5408
5409     my $volid_hash = scan_volids($cfg, $vmid);
5410
5411     my $updatefn =  sub {
5412         my ($vmid) = @_;
5413
5414         my $conf = PVE::QemuConfig->load_config($vmid);
5415
5416         PVE::QemuConfig->check_lock($conf);
5417
5418         my $vm_volids = {};
5419         foreach my $volid (keys %$volid_hash) {
5420             my $info = $volid_hash->{$volid};
5421             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5422         }
5423
5424         my $changes = update_disksize($vmid, $conf, $vm_volids);
5425
5426         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5427     };
5428
5429     if (defined($vmid)) {
5430         if ($nolock) {
5431             &$updatefn($vmid);
5432         } else {
5433             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5434         }
5435     } else {
5436         my $vmlist = config_list();
5437         foreach my $vmid (keys %$vmlist) {
5438             if ($nolock) {
5439                 &$updatefn($vmid);
5440             } else {
5441                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5442             }
5443         }
5444     }
5445 }
5446
5447 sub restore_vma_archive {
5448     my ($archive, $vmid, $user, $opts, $comp) = @_;
5449
5450     my $input = $archive eq '-' ? "<&STDIN" : undef;
5451     my $readfrom = $archive;
5452
5453     my $uncomp = '';
5454     if ($comp) {
5455         $readfrom = '-';
5456         my $qarchive = PVE::Tools::shellquote($archive);
5457         if ($comp eq 'gzip') {
5458             $uncomp = "zcat $qarchive|";
5459         } elsif ($comp eq 'lzop') {
5460             $uncomp = "lzop -d -c $qarchive|";
5461         } else {
5462             die "unknown compression method '$comp'\n";
5463         }
5464
5465     }
5466
5467     my $tmpdir = "/var/tmp/vzdumptmp$$";
5468     rmtree $tmpdir;
5469
5470     # disable interrupts (always do cleanups)
5471     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5472         warn "got interrupt - ignored\n";
5473     };
5474
5475     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5476     POSIX::mkfifo($mapfifo, 0600);
5477     my $fifofh;
5478
5479     my $openfifo = sub {
5480         open($fifofh, '>', $mapfifo) || die $!;
5481     };
5482
5483     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5484
5485     my $oldtimeout;
5486     my $timeout = 5;
5487
5488     my $devinfo = {};
5489
5490     my $rpcenv = PVE::RPCEnvironment::get();
5491
5492     my $conffile = PVE::QemuConfig->config_file($vmid);
5493     my $tmpfn = "$conffile.$$.tmp";
5494
5495     # Note: $oldconf is undef if VM does not exists
5496     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5497     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5498
5499     my $print_devmap = sub {
5500         my $virtdev_hash = {};
5501
5502         my $cfgfn = "$tmpdir/qemu-server.conf";
5503
5504         # we can read the config - that is already extracted
5505         my $fh = IO::File->new($cfgfn, "r") ||
5506             "unable to read qemu-server.conf - $!\n";
5507
5508         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5509         if (-f $fwcfgfn) {
5510             my $pve_firewall_dir = '/etc/pve/firewall';
5511             mkdir $pve_firewall_dir; # make sure the dir exists
5512             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5513         }
5514
5515         while (defined(my $line = <$fh>)) {
5516             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5517                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5518                 die "archive does not contain data for drive '$virtdev'\n"
5519                     if !$devinfo->{$devname};
5520                 if (defined($opts->{storage})) {
5521                     $storeid = $opts->{storage} || 'local';
5522                 } elsif (!$storeid) {
5523                     $storeid = 'local';
5524                 }
5525                 $format = 'raw' if !$format;
5526                 $devinfo->{$devname}->{devname} = $devname;
5527                 $devinfo->{$devname}->{virtdev} = $virtdev;
5528                 $devinfo->{$devname}->{format} = $format;
5529                 $devinfo->{$devname}->{storeid} = $storeid;
5530
5531                 # check permission on storage
5532                 my $pool = $opts->{pool}; # todo: do we need that?
5533                 if ($user ne 'root@pam') {
5534                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5535                 }
5536
5537                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5538             }
5539         }
5540
5541         foreach my $devname (keys %$devinfo) {
5542             die "found no device mapping information for device '$devname'\n"
5543                 if !$devinfo->{$devname}->{virtdev};
5544         }
5545
5546         my $cfg = PVE::Storage::config();
5547
5548         # create empty/temp config
5549         if ($oldconf) {
5550             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5551             foreach_drive($oldconf, sub {
5552                 my ($ds, $drive) = @_;
5553
5554                 return if drive_is_cdrom($drive);
5555
5556                 my $volid = $drive->{file};
5557
5558                 return if !$volid || $volid =~ m|^/|;
5559
5560                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5561                 return if !$path || !$owner || ($owner != $vmid);
5562
5563                 # Note: only delete disk we want to restore
5564                 # other volumes will become unused
5565                 if ($virtdev_hash->{$ds}) {
5566                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5567                     if (my $err = $@) {
5568                         warn $err;
5569                     }
5570                 }
5571             });
5572
5573             # delete vmstate files
5574             # since after the restore we have no snapshots anymore
5575             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5576                 my $snap = $oldconf->{snapshots}->{$snapname};
5577                 if ($snap->{vmstate}) {
5578                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5579                     if (my $err = $@) {
5580                         warn $err;
5581                     }
5582                 }
5583             }
5584         }
5585
5586         my $map = {};
5587         foreach my $virtdev (sort keys %$virtdev_hash) {
5588             my $d = $virtdev_hash->{$virtdev};
5589             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5590             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5591
5592             # test if requested format is supported
5593             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5594             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5595             $d->{format} = $defFormat if !$supported;
5596
5597             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5598                                                   $d->{format}, undef, $alloc_size);
5599             print STDERR "new volume ID is '$volid'\n";
5600             $d->{volid} = $volid;
5601             my $path = PVE::Storage::path($cfg, $volid);
5602
5603             PVE::Storage::activate_volumes($cfg,[$volid]);
5604
5605             my $write_zeros = 1;
5606             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5607                 $write_zeros = 0;
5608             }
5609
5610             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5611
5612             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5613             $map->{$virtdev} = $volid;
5614         }
5615
5616         $fh->seek(0, 0) || die "seek failed - $!\n";
5617
5618         my $outfd = new IO::File ($tmpfn, "w") ||
5619             die "unable to write config for VM $vmid\n";
5620
5621         my $cookie = { netcount => 0 };
5622         while (defined(my $line = <$fh>)) {
5623             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5624         }
5625
5626         $fh->close();
5627         $outfd->close();
5628     };
5629
5630     eval {
5631         # enable interrupts
5632         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5633             die "interrupted by signal\n";
5634         };
5635         local $SIG{ALRM} = sub { die "got timeout\n"; };
5636
5637         $oldtimeout = alarm($timeout);
5638
5639         my $parser = sub {
5640             my $line = shift;
5641
5642             print "$line\n";
5643
5644             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5645                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5646                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5647             } elsif ($line =~ m/^CTIME: /) {
5648                 # we correctly received the vma config, so we can disable
5649                 # the timeout now for disk allocation (set to 10 minutes, so
5650                 # that we always timeout if something goes wrong)
5651                 alarm(600);
5652                 &$print_devmap();
5653                 print $fifofh "done\n";
5654                 my $tmp = $oldtimeout || 0;
5655                 $oldtimeout = undef;
5656                 alarm($tmp);
5657                 close($fifofh);
5658             }
5659         };
5660
5661         print "restore vma archive: $cmd\n";
5662         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5663     };
5664     my $err = $@;
5665
5666     alarm($oldtimeout) if $oldtimeout;
5667
5668     my $vollist = [];
5669     foreach my $devname (keys %$devinfo) {
5670         my $volid = $devinfo->{$devname}->{volid};
5671         push @$vollist, $volid if $volid;
5672     }
5673
5674     my $cfg = PVE::Storage::config();
5675     PVE::Storage::deactivate_volumes($cfg, $vollist);
5676
5677     unlink $mapfifo;
5678
5679     if ($err) {
5680         rmtree $tmpdir;
5681         unlink $tmpfn;
5682
5683         foreach my $devname (keys %$devinfo) {
5684             my $volid = $devinfo->{$devname}->{volid};
5685             next if !$volid;
5686             eval {
5687                 if ($volid =~ m|^/|) {
5688                     unlink $volid || die 'unlink failed\n';
5689                 } else {
5690                     PVE::Storage::vdisk_free($cfg, $volid);
5691                 }
5692                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5693             };
5694             print STDERR "unable to cleanup '$volid' - $@" if $@;
5695         }
5696         die $err;
5697     }
5698
5699     rmtree $tmpdir;
5700
5701     rename($tmpfn, $conffile) ||
5702         die "unable to commit configuration file '$conffile'\n";
5703
5704     PVE::Cluster::cfs_update(); # make sure we read new file
5705
5706     eval { rescan($vmid, 1); };
5707     warn $@ if $@;
5708 }
5709
5710 sub restore_tar_archive {
5711     my ($archive, $vmid, $user, $opts) = @_;
5712
5713     if ($archive ne '-') {
5714         my $firstfile = tar_archive_read_firstfile($archive);
5715         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5716             if $firstfile ne 'qemu-server.conf';
5717     }
5718
5719     my $storecfg = PVE::Storage::config();
5720
5721     # destroy existing data - keep empty config
5722     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5723     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5724
5725     my $tocmd = "/usr/lib/qemu-server/qmextract";
5726
5727     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5728     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5729     $tocmd .= ' --prealloc' if $opts->{prealloc};
5730     $tocmd .= ' --info' if $opts->{info};
5731
5732     # tar option "xf" does not autodetect compression when read from STDIN,
5733     # so we pipe to zcat
5734     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5735         PVE::Tools::shellquote("--to-command=$tocmd");
5736
5737     my $tmpdir = "/var/tmp/vzdumptmp$$";
5738     mkpath $tmpdir;
5739
5740     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5741     local $ENV{VZDUMP_VMID} = $vmid;
5742     local $ENV{VZDUMP_USER} = $user;
5743
5744     my $conffile = PVE::QemuConfig->config_file($vmid);
5745     my $tmpfn = "$conffile.$$.tmp";
5746
5747     # disable interrupts (always do cleanups)
5748     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5749         print STDERR "got interrupt - ignored\n";
5750     };
5751
5752     eval {
5753         # enable interrupts
5754         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5755             die "interrupted by signal\n";
5756         };
5757
5758         if ($archive eq '-') {
5759             print "extracting archive from STDIN\n";
5760             run_command($cmd, input => "<&STDIN");
5761         } else {
5762             print "extracting archive '$archive'\n";
5763             run_command($cmd);
5764         }
5765
5766         return if $opts->{info};
5767
5768         # read new mapping
5769         my $map = {};
5770         my $statfile = "$tmpdir/qmrestore.stat";
5771         if (my $fd = IO::File->new($statfile, "r")) {
5772             while (defined (my $line = <$fd>)) {
5773                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5774                     $map->{$1} = $2 if $1;
5775                 } else {
5776                     print STDERR "unable to parse line in statfile - $line\n";
5777                 }
5778             }
5779             $fd->close();
5780         }
5781
5782         my $confsrc = "$tmpdir/qemu-server.conf";
5783
5784         my $srcfd = new IO::File($confsrc, "r") ||
5785             die "unable to open file '$confsrc'\n";
5786
5787         my $outfd = new IO::File ($tmpfn, "w") ||
5788             die "unable to write config for VM $vmid\n";
5789
5790         my $cookie = { netcount => 0 };
5791         while (defined (my $line = <$srcfd>)) {
5792             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5793         }
5794
5795         $srcfd->close();
5796         $outfd->close();
5797     };
5798     my $err = $@;
5799
5800     if ($err) {
5801
5802         unlink $tmpfn;
5803
5804         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5805
5806         die $err;
5807     }
5808
5809     rmtree $tmpdir;
5810
5811     rename $tmpfn, $conffile ||
5812         die "unable to commit configuration file '$conffile'\n";
5813
5814     PVE::Cluster::cfs_update(); # make sure we read new file
5815
5816     eval { rescan($vmid, 1); };
5817     warn $@ if $@;
5818 };
5819
5820 sub foreach_writable_storage {
5821     my ($conf, $func) = @_;
5822
5823     my $sidhash = {};
5824
5825     foreach my $ds (keys %$conf) {
5826         next if !is_valid_drivename($ds);
5827
5828         my $drive = parse_drive($ds, $conf->{$ds});
5829         next if !$drive;
5830         next if drive_is_cdrom($drive);
5831
5832         my $volid = $drive->{file};
5833
5834         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5835         $sidhash->{$sid} = $sid if $sid;
5836     }
5837
5838     foreach my $sid (sort keys %$sidhash) {
5839         &$func($sid);
5840     }
5841 }
5842
5843 sub do_snapshots_with_qemu {
5844     my ($storecfg, $volid) = @_;
5845
5846     my $storage_name = PVE::Storage::parse_volume_id($volid);
5847
5848     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5849         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5850         return 1;
5851     }
5852
5853     if ($volid =~ m/\.(qcow2|qed)$/){
5854         return 1;
5855     }
5856
5857     return undef;
5858 }
5859
5860 sub qga_check_running {
5861     my ($vmid) = @_;
5862
5863     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5864     if ($@) {
5865         warn "Qemu Guest Agent are not running - $@";
5866         return 0;
5867     }
5868     return 1;
5869 }
5870
5871 sub template_create {
5872     my ($vmid, $conf, $disk) = @_;
5873
5874     my $storecfg = PVE::Storage::config();
5875
5876     foreach_drive($conf, sub {
5877         my ($ds, $drive) = @_;
5878
5879         return if drive_is_cdrom($drive);
5880         return if $disk && $ds ne $disk;
5881
5882         my $volid = $drive->{file};
5883         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5884
5885         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5886         $drive->{file} = $voliddst;
5887         $conf->{$ds} = print_drive($vmid, $drive);
5888         PVE::QemuConfig->write_config($vmid, $conf);
5889     });
5890 }
5891
5892 sub qemu_img_convert {
5893     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5894
5895     my $storecfg = PVE::Storage::config();
5896     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5897     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5898
5899     if ($src_storeid && $dst_storeid) {
5900
5901         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5902
5903         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5904         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5905
5906         my $src_format = qemu_img_format($src_scfg, $src_volname);
5907         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5908
5909         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5910         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5911
5912         my $cmd = [];
5913         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5914         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5915         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5916         if ($is_zero_initialized) {
5917             push @$cmd, "zeroinit:$dst_path";
5918         } else {
5919             push @$cmd, $dst_path;
5920         }
5921
5922         my $parser = sub {
5923             my $line = shift;
5924             if($line =~ m/\((\S+)\/100\%\)/){
5925                 my $percent = $1;
5926                 my $transferred = int($size * $percent / 100);
5927                 my $remaining = $size - $transferred;
5928
5929                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5930             }
5931
5932         };
5933
5934         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5935         my $err = $@;
5936         die "copy failed: $err" if $err;
5937     }
5938 }
5939
5940 sub qemu_img_format {
5941     my ($scfg, $volname) = @_;
5942
5943     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5944         return $1;
5945     } else {
5946         return "raw";
5947     }
5948 }
5949
5950 sub qemu_drive_mirror {
5951     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
5952
5953     $jobs = {} if !$jobs;
5954
5955     my $qemu_target;
5956     my $format;
5957     $jobs->{"drive-$drive"} = {};
5958
5959     if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
5960         my $server = $1;
5961         my $port = $2;
5962         my $exportname = $3;
5963
5964         $format = "nbd";
5965         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
5966         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
5967         my $cmd = ['socat', '-T30', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=5"];
5968
5969         my $pid = fork();
5970         if (!defined($pid)) {
5971             die "forking socat tunnel failed\n";
5972         } elsif ($pid == 0) {
5973             exec(@$cmd);
5974             warn "exec failed: $!\n";
5975             POSIX::_exit(-1);
5976         }
5977         $jobs->{"drive-$drive"}->{pid} = $pid;
5978
5979         my $timeout = 0;
5980         while (!-S $unixsocket) {
5981             die "nbd connection helper timed out\n"
5982                 if $timeout++ > 5;
5983             sleep 1;
5984         }
5985     } else {
5986         my $storecfg = PVE::Storage::config();
5987         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5988
5989         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5990
5991         $format = qemu_img_format($dst_scfg, $dst_volname);
5992
5993         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5994
5995         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5996     }
5997
5998     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5999     $opts->{format} = $format if $format;
6000
6001     print "drive mirror is starting for drive-$drive\n";
6002
6003     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6004
6005     if (my $err = $@) {
6006         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6007         die "mirroring error: $err";
6008     }
6009
6010     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6011 }
6012
6013 sub qemu_drive_mirror_monitor {
6014     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6015
6016     eval {
6017         my $err_complete = 0;
6018
6019         while (1) {
6020             die "storage migration timed out\n" if $err_complete > 300;
6021
6022             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6023
6024             my $running_mirror_jobs = {};
6025             foreach my $stat (@$stats) {
6026                 next if $stat->{type} ne 'mirror';
6027                 $running_mirror_jobs->{$stat->{device}} = $stat;
6028             }
6029
6030             my $readycounter = 0;
6031
6032             foreach my $job (keys %$jobs) {
6033
6034                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6035                     print "$job : finished\n";
6036                     delete $jobs->{$job};
6037                     next;
6038                 }
6039
6040                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6041
6042                 my $busy = $running_mirror_jobs->{$job}->{busy};
6043                 my $ready = $running_mirror_jobs->{$job}->{ready};
6044                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6045                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6046                     my $remaining = $total - $transferred;
6047                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6048
6049                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6050                 }
6051
6052                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6053             }
6054
6055             last if scalar(keys %$jobs) == 0;
6056
6057             if ($readycounter == scalar(keys %$jobs)) {
6058                 print "all mirroring jobs are ready \n";
6059                 last if $skipcomplete; #do the complete later
6060
6061                 if ($vmiddst && $vmiddst != $vmid) {
6062                     if ($qga) {
6063                         print "freeze filesystem\n";
6064                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6065                     } else {
6066                         print "suspend vm\n";
6067                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6068                     }
6069
6070                     # if we clone a disk for a new target vm, we don't switch the disk
6071                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6072
6073                     if ($qga) {
6074                         print "unfreeze filesystem\n";
6075                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6076                     } else {
6077                         print "resume vm\n";
6078                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6079                     }
6080
6081                     last;
6082                 } else {
6083
6084                     foreach my $job (keys %$jobs) {
6085                         # try to switch the disk if source and destination are on the same guest
6086                         print "$job: Completing block job...\n";
6087
6088                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6089                         if ($@ =~ m/cannot be completed/) {
6090                             print "$job: Block job cannot be completed, try again.\n";
6091                             $err_complete++;
6092                         }else {
6093                             print "$job: Completed successfully.\n";
6094                             $jobs->{$job}->{complete} = 1;
6095                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6096                         }
6097                     }
6098                 }
6099             }
6100             sleep 1;
6101         }
6102     };
6103     my $err = $@;
6104
6105     if ($err) {
6106         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6107         die "mirroring error: $err";
6108     }
6109
6110 }
6111
6112 sub qemu_blockjobs_cancel {
6113     my ($vmid, $jobs) = @_;
6114
6115     foreach my $job (keys %$jobs) {
6116         print "$job: Cancelling block job\n";
6117         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6118         $jobs->{$job}->{cancel} = 1;
6119     }
6120
6121     while (1) {
6122         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6123
6124         my $running_jobs = {};
6125         foreach my $stat (@$stats) {
6126             $running_jobs->{$stat->{device}} = $stat;
6127         }
6128
6129         foreach my $job (keys %$jobs) {
6130
6131             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6132                 print "$job: Done.\n";
6133                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6134                 delete $jobs->{$job};
6135             }
6136         }
6137
6138         last if scalar(keys %$jobs) == 0;
6139
6140         sleep 1;
6141     }
6142 }
6143
6144 sub qemu_blockjobs_finish_tunnel {
6145    my ($vmid, $job, $cpid) = @_;
6146
6147    return if !$cpid;
6148
6149    for (my $i = 1; $i < 20; $i++) {
6150         my $waitpid = waitpid($cpid, WNOHANG);
6151         last if (defined($waitpid) && ($waitpid == $cpid));
6152
6153         if ($i == 10) {
6154             kill(15, $cpid);
6155         } elsif ($i >= 15) {
6156             kill(9, $cpid);
6157         }
6158         sleep (1);
6159     }
6160     unlink "/run/qemu-server/$vmid.mirror-$job";
6161 }
6162
6163 sub clone_disk {
6164     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6165         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6166
6167     my $newvolid;
6168
6169     if (!$full) {
6170         print "create linked clone of drive $drivename ($drive->{file})\n";
6171         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6172         push @$newvollist, $newvolid;
6173     } else {
6174
6175         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6176         $storeid = $storage if $storage;
6177
6178         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6179         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6180
6181         print "create full clone of drive $drivename ($drive->{file})\n";
6182         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, undef, ($size/1024));
6183         push @$newvollist, $newvolid;
6184
6185         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6186
6187         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6188         if (!$running || $snapname) {
6189             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6190         } else {
6191
6192             my $kvmver = get_running_qemu_version ($vmid);
6193             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6194                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6195                     if $drive->{iothread};
6196             }
6197
6198             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6199         }
6200     }
6201
6202     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6203
6204     my $disk = $drive;
6205     $disk->{format} = undef;
6206     $disk->{file} = $newvolid;
6207     $disk->{size} = $size;
6208
6209     return $disk;
6210 }
6211
6212 # this only works if VM is running
6213 sub get_current_qemu_machine {
6214     my ($vmid) = @_;
6215
6216     my $cmd = { execute => 'query-machines', arguments => {} };
6217     my $res = vm_qmp_command($vmid, $cmd);
6218
6219     my ($current, $default);
6220     foreach my $e (@$res) {
6221         $default = $e->{name} if $e->{'is-default'};
6222         $current = $e->{name} if $e->{'is-current'};
6223     }
6224
6225     # fallback to the default machine if current is not supported by qemu
6226     return $current || $default || 'pc';
6227 }
6228
6229 sub get_running_qemu_version {
6230     my ($vmid) = @_;
6231     my $cmd = { execute => 'query-version', arguments => {} };
6232     my $res = vm_qmp_command($vmid, $cmd);
6233     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6234 }
6235
6236 sub qemu_machine_feature_enabled {
6237     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6238
6239     my $current_major;
6240     my $current_minor;
6241
6242     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6243
6244         $current_major = $3;
6245         $current_minor = $4;
6246
6247     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6248
6249         $current_major = $1;
6250         $current_minor = $2;
6251     }
6252
6253     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6254
6255
6256 }
6257
6258 sub qemu_machine_pxe {
6259     my ($vmid, $conf, $machine) = @_;
6260
6261     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6262
6263     foreach my $opt (keys %$conf) {
6264         next if $opt !~ m/^net(\d+)$/;
6265         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6266         next if !$net;
6267         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6268         return $machine.".pxe" if $romfile =~ m/pxe/;
6269         last;
6270     }
6271
6272     return $machine;
6273 }
6274
6275 sub qemu_use_old_bios_files {
6276     my ($machine_type) = @_;
6277
6278     return if !$machine_type;
6279
6280     my $use_old_bios_files = undef;
6281
6282     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6283         $machine_type = $1;
6284         $use_old_bios_files = 1;
6285     } else {
6286         my $kvmver = kvm_user_version();
6287         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6288         # load new efi bios files on migration. So this hack is required to allow
6289         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6290         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6291         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6292     }
6293
6294     return ($use_old_bios_files, $machine_type);
6295 }
6296
6297 sub lspci {
6298
6299     my $devices = {};
6300
6301     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6302             my (undef, $id, $function) = @_;
6303             my $res = { id => $id, function => $function};
6304             push @{$devices->{$id}}, $res;
6305     });
6306
6307     return $devices;
6308 }
6309
6310 sub vm_iothreads_list {
6311     my ($vmid) = @_;
6312
6313     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6314
6315     my $iothreads = {};
6316     foreach my $iothread (@$res) {
6317         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6318     }
6319
6320     return $iothreads;
6321 }
6322
6323 sub scsihw_infos {
6324     my ($conf, $drive) = @_;
6325
6326     my $maxdev = 0;
6327
6328     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6329         $maxdev = 7;
6330     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6331         $maxdev = 1;
6332     } else {
6333         $maxdev = 256;
6334     }
6335
6336     my $controller = int($drive->{index} / $maxdev);
6337     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6338
6339     return ($maxdev, $controller, $controller_prefix);
6340 }
6341
6342 sub add_hyperv_enlighments {
6343     my ($cpuFlags, $winversion, $machine_type, $kvmver, $nokvm, $bios, $gpu_passthrough) = @_;
6344
6345     return if $nokvm;
6346     return if $winversion < 6;
6347     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6348
6349     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6350
6351     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6352         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6353         push @$cpuFlags , 'hv_vapic';
6354         push @$cpuFlags , 'hv_time';
6355     } else {
6356         push @$cpuFlags , 'hv_spinlocks=0xffff';
6357     }
6358
6359     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6360         push @$cpuFlags , 'hv_reset';
6361         push @$cpuFlags , 'hv_vpindex';
6362         push @$cpuFlags , 'hv_runtime';
6363     }
6364
6365     if ($winversion >= 7) {
6366         push @$cpuFlags , 'hv_relaxed';
6367     }
6368 }
6369
6370 sub windows_version {
6371     my ($ostype) = @_;
6372
6373     return 0 if !$ostype;
6374
6375     my $winversion = 0;
6376
6377     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6378         $winversion = 5;
6379     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6380         $winversion = 6;
6381     } elsif ($ostype =~ m/^win(\d+)$/) {
6382         $winversion = $1;
6383     }
6384
6385     return $winversion;
6386 }
6387
6388 sub resolve_dst_disk_format {
6389         my ($storecfg, $storeid, $src_volname, $format) = @_;
6390         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6391
6392         if (!$format) {
6393             # if no target format is specified, use the source disk format as hint
6394             if ($src_volname) {
6395                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6396                 $format = qemu_img_format($scfg, $src_volname);
6397             } else {
6398                 return $defFormat;
6399             }
6400         }
6401
6402         # test if requested format is supported - else use default
6403         my $supported = grep { $_ eq $format } @$validFormats;
6404         $format = $defFormat if !$supported;
6405         return $format;
6406 }
6407
6408 # bash completion helper
6409
6410 sub complete_backup_archives {
6411     my ($cmdname, $pname, $cvalue) = @_;
6412
6413     my $cfg = PVE::Storage::config();
6414
6415     my $storeid;
6416
6417     if ($cvalue =~ m/^([^:]+):/) {
6418         $storeid = $1;
6419     }
6420
6421     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6422
6423     my $res = [];
6424     foreach my $id (keys %$data) {
6425         foreach my $item (@{$data->{$id}}) {
6426             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6427             push @$res, $item->{volid} if defined($item->{volid});
6428         }
6429     }
6430
6431     return $res;
6432 }
6433
6434 my $complete_vmid_full = sub {
6435     my ($running) = @_;
6436
6437     my $idlist = vmstatus();
6438
6439     my $res = [];
6440
6441     foreach my $id (keys %$idlist) {
6442         my $d = $idlist->{$id};
6443         if (defined($running)) {
6444             next if $d->{template};
6445             next if $running && $d->{status} ne 'running';
6446             next if !$running && $d->{status} eq 'running';
6447         }
6448         push @$res, $id;
6449
6450     }
6451     return $res;
6452 };
6453
6454 sub complete_vmid {
6455     return &$complete_vmid_full();
6456 }
6457
6458 sub complete_vmid_stopped {
6459     return &$complete_vmid_full(0);
6460 }
6461
6462 sub complete_vmid_running {
6463     return &$complete_vmid_full(1);
6464 }
6465
6466 sub complete_storage {
6467
6468     my $cfg = PVE::Storage::config();
6469     my $ids = $cfg->{ids};
6470
6471     my $res = [];
6472     foreach my $sid (keys %$ids) {
6473         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6474         next if !$ids->{$sid}->{content}->{images};
6475         push @$res, $sid;
6476     }
6477
6478     return $res;
6479 }
6480
6481 sub nbd_stop {
6482     my ($vmid) = @_;
6483
6484     vm_mon_cmd($vmid, 'nbd-server-stop');
6485 }
6486
6487 1;