PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42 my $OVMF_IMG = '/usr/share/kvm/OVMF-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 # Note about locking: we use flock on the config file protect
  49 # against concurent actions.
  50 # Aditionaly, we have a 'lock' setting in the config file. This
  51 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  52 # allowed when such lock is set. But you can ignore this kind of
  53 # lock with the --skiplock flag.
  54
  55 cfs_register_file('/qemu-server/',
  56                   \&parse_vm_config,
  57                   \&write_vm_config);
  58
  59 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  60     description => "Some command save/restore state from this location.",
  61     type => 'string',
  62     maxLength => 128,
  63     optional => 1,
  64 });
  65
  66 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  67     description => "The name of the snapshot.",
  68     type => 'string', format => 'pve-configid',
  69     maxLength => 40,
  70 });
  71
  72 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  73     type => 'string',
  74     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  75     description => "The drive's backing file's data format.",
  76     optional => 1,
  77 });
  78
  79 #no warnings 'redefine';
  80
  81 sub cgroups_write {
  82    my ($controller, $vmid, $option, $value) = @_;
  83
  84    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  85    PVE::ProcFSTools::write_proc_entry($path, $value);
  86
  87 }
  88
  89 my $nodename = PVE::INotify::nodename();
  90
  91 mkdir "/etc/pve/nodes/$nodename";
  92 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  93 mkdir $confdir;
  94
  95 my $var_run_tmpdir = "/var/run/qemu-server";
  96 mkdir $var_run_tmpdir;
  97
  98 my $lock_dir = "/var/lock/qemu-server";
  99 mkdir $lock_dir;
 100
 101 my $pcisysfs = "/sys/bus/pci";
 102
 103 my $cpu_vendor_list = {
 104     # Intel CPUs
 105     486 => 'GenuineIntel',
 106     pentium => 'GenuineIntel',
 107     pentium2  => 'GenuineIntel',
 108     pentium3  => 'GenuineIntel',
 109     coreduo => 'GenuineIntel',
 110     core2duo => 'GenuineIntel',
 111     Conroe  => 'GenuineIntel',
 112     Penryn  => 'GenuineIntel',
 113     Nehalem  => 'GenuineIntel',
 114     Westmere => 'GenuineIntel',
 115     SandyBridge => 'GenuineIntel',
 116     IvyBridge => 'GenuineIntel',
 117     Haswell => 'GenuineIntel',
 118     'Haswell-noTSX' => 'GenuineIntel',
 119     Broadwell => 'GenuineIntel',
 120     'Broadwell-noTSX' => 'GenuineIntel',
 121     'Skylake-Client' => 'GenuineIntel',
 122
 123     # AMD CPUs
 124     athlon => 'AuthenticAMD',
 125     phenom  => 'AuthenticAMD',
 126     Opteron_G1  => 'AuthenticAMD',
 127     Opteron_G2  => 'AuthenticAMD',
 128     Opteron_G3  => 'AuthenticAMD',
 129     Opteron_G4  => 'AuthenticAMD',
 130     Opteron_G5  => 'AuthenticAMD',
 131
 132     # generic types, use vendor from host node
 133     host => 'default',
 134     kvm32 => 'default',
 135     kvm64 => 'default',
 136     qemu32 => 'default',
 137     qemu64 => 'default',
 138 };
 139
 140 my $cpu_fmt = {
 141     cputype => {
 142         description => "Emulated CPU type.",
 143         type => 'string',
 144         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 145         default => 'kvm64',
 146         default_key => 1,
 147     },
 148     hidden => {
 149         description => "Do not identify as a KVM virtual machine.",
 150         type => 'boolean',
 151         optional => 1,
 152         default => 0
 153     },
 154 };
 155
 156 my $watchdog_fmt = {
 157     model => {
 158         default_key => 1,
 159         type => 'string',
 160         enum => [qw(i6300esb ib700)],
 161         description => "Watchdog type to emulate.",
 162         default => 'i6300esb',
 163         optional => 1,
 164     },
 165     action => {
 166         type => 'string',
 167         enum => [qw(reset shutdown poweroff pause debug none)],
 168         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 169         optional => 1,
 170     },
 171 };
 172 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 173
 174 my $confdesc = {
 175     onboot => {
 176         optional => 1,
 177         type => 'boolean',
 178         description => "Specifies whether a VM will be started during system bootup.",
 179         default => 0,
 180     },
 181     autostart => {
 182         optional => 1,
 183         type => 'boolean',
 184         description => "Automatic restart after crash (currently ignored).",
 185         default => 0,
 186     },
 187     hotplug => {
 188         optional => 1,
 189         type => 'string', format => 'pve-hotplug-features',
 190         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 191         default => 'network,disk,usb',
 192     },
 193     reboot => {
 194         optional => 1,
 195         type => 'boolean',
 196         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 197         default => 1,
 198     },
 199     lock => {
 200         optional => 1,
 201         type => 'string',
 202         description => "Lock/unlock the VM.",
 203         enum => [qw(migrate backup snapshot rollback)],
 204     },
 205     cpulimit => {
 206         optional => 1,
 207         type => 'number',
 208         description => "Limit of CPU usage.",
 209         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 210         minimum => 0,
 211         maximum => 128,
 212         default => 0,
 213     },
 214     cpuunits => {
 215         optional => 1,
 216         type => 'integer',
 217         description => "CPU weight for a VM.",
 218         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 219         minimum => 0,
 220         maximum => 500000,
 221         default => 1024,
 222     },
 223     memory => {
 224         optional => 1,
 225         type => 'integer',
 226         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 227         minimum => 16,
 228         default => 512,
 229     },
 230     balloon => {
 231         optional => 1,
 232         type => 'integer',
 233         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 234         minimum => 0,
 235     },
 236     shares => {
 237         optional => 1,
 238         type => 'integer',
 239         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 240         minimum => 0,
 241         maximum => 50000,
 242         default => 1000,
 243     },
 244     keyboard => {
 245         optional => 1,
 246         type => 'string',
 247         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 248         enum => PVE::Tools::kvmkeymaplist(),
 249         default => 'en-us',
 250     },
 251     name => {
 252         optional => 1,
 253         type => 'string', format => 'dns-name',
 254         description => "Set a name for the VM. Only used on the configuration web interface.",
 255     },
 256     scsihw => {
 257         optional => 1,
 258         type => 'string',
 259         description => "SCSI controller model",
 260         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 261         default => 'lsi',
 262     },
 263     description => {
 264         optional => 1,
 265         type => 'string',
 266         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 267     },
 268     ostype => {
 269         optional => 1,
 270         type => 'string',
 271         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 272         description => "Specify guest operating system.",
 273         verbose_description => <<EODESC,
 274 Specify guest operating system. This is used to enable special
 275 optimization/features for specific operating systems:
 276
 277 [horizontal]
 278 other;; unspecified OS
 279 wxp;; Microsoft Windows XP
 280 w2k;; Microsoft Windows 2000
 281 w2k3;; Microsoft Windows 2003
 282 w2k8;; Microsoft Windows 2008
 283 wvista;; Microsoft Windows Vista
 284 win7;; Microsoft Windows 7
 285 win8;; Microsoft Windows 8/2012
 286 l24;; Linux 2.4 Kernel
 287 l26;; Linux 2.6/3.X Kernel
 288 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 289 EODESC
 290     },
 291     boot => {
 292         optional => 1,
 293         type => 'string',
 294         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 295         pattern => '[acdn]{1,4}',
 296         default => 'cdn',
 297     },
 298     bootdisk => {
 299         optional => 1,
 300         type => 'string', format => 'pve-qm-bootdisk',
 301         description => "Enable booting from specified disk.",
 302         pattern => '(ide|sata|scsi|virtio)\d+',
 303     },
 304     smp => {
 305         optional => 1,
 306         type => 'integer',
 307         description => "The number of CPUs. Please use option -sockets instead.",
 308         minimum => 1,
 309         default => 1,
 310     },
 311     sockets => {
 312         optional => 1,
 313         type => 'integer',
 314         description => "The number of CPU sockets.",
 315         minimum => 1,
 316         default => 1,
 317     },
 318     cores => {
 319         optional => 1,
 320         type => 'integer',
 321         description => "The number of cores per socket.",
 322         minimum => 1,
 323         default => 1,
 324     },
 325     numa => {
 326         optional => 1,
 327         type => 'boolean',
 328         description => "Enable/disable NUMA.",
 329         default => 0,
 330     },
 331     hugepages => {
 332         optional => 1,
 333         type => 'string',
 334         description => "Enable/disable hugepages memory.",
 335         enum => [qw(any 2 1024)],
 336     },
 337     vcpus => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "Number of hotplugged vcpus.",
 341         minimum => 1,
 342         default => 0,
 343     },
 344     acpi => {
 345         optional => 1,
 346         type => 'boolean',
 347         description => "Enable/disable ACPI.",
 348         default => 1,
 349     },
 350     agent => {
 351         optional => 1,
 352         type => 'boolean',
 353         description => "Enable/disable Qemu GuestAgent.",
 354         default => 0,
 355     },
 356     kvm => {
 357         optional => 1,
 358         type => 'boolean',
 359         description => "Enable/disable KVM hardware virtualization.",
 360         default => 1,
 361     },
 362     tdf => {
 363         optional => 1,
 364         type => 'boolean',
 365         description => "Enable/disable time drift fix.",
 366         default => 0,
 367     },
 368     localtime => {
 369         optional => 1,
 370         type => 'boolean',
 371         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 372     },
 373     freeze => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 377     },
 378     vga => {
 379         optional => 1,
 380         type => 'string',
 381         description => "Select the VGA type.",
 382         verbose_description => "Select the VGA type. If you want to use high resolution" .
 383             " modes (>= 1280x1024x16) then you should use the options " .
 384             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 385             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 386             "display sever. For win* OS you can select how many independent " .
 387             "displays you want, Linux guests can add displays them self. " .
 388             "You can also run without any graphic card, using a serial device" .
 389             " as terminal.",
 390         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 391     },
 392     watchdog => {
 393         optional => 1,
 394         type => 'string', format => 'pve-qm-watchdog',
 395         description => "Create a virtual hardware watchdog device.",
 396         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 397             " (by a guest action), the watchdog must be periodically polled " .
 398             "by an agent inside the guest or else the watchdog will reset " .
 399             "the guest (or execute the respective action specified)",
 400     },
 401     startdate => {
 402         optional => 1,
 403         type => 'string',
 404         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 405         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 406         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 407         default => 'now',
 408     },
 409     startup =>  get_standard_option('pve-startup-order'),
 410     template => {
 411         optional => 1,
 412         type => 'boolean',
 413         description => "Enable/disable Template.",
 414         default => 0,
 415     },
 416     args => {
 417         optional => 1,
 418         type => 'string',
 419         description => "Arbitrary arguments passed to kvm.",
 420         verbose_description => <<EODESCR,
 421 Arbitrary arguments passed to kvm, for example:
 422
 423 args: -no-reboot -no-hpet
 424
 425 NOTE: this option is for experts only.
 426 EODESCR
 427     },
 428     tablet => {
 429         optional => 1,
 430         type => 'boolean',
 431         default => 1,
 432         description => "Enable/disable the USB tablet device.",
 433         verbose_description => "Enable/disable the USB tablet device. This device is " .
 434             "usually needed to allow absolute mouse positioning with VNC. " .
 435             "Else the mouse runs out of sync with normal VNC clients. " .
 436             "If you're running lots of console-only guests on one host, " .
 437             "you may consider disabling this to save some context switches. " .
 438             "This is turned off by default if you use spice (-vga=qxl).",
 439     },
 440     migrate_speed => {
 441         optional => 1,
 442         type => 'integer',
 443         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 444         minimum => 0,
 445         default => 0,
 446     },
 447     migrate_downtime => {
 448         optional => 1,
 449         type => 'number',
 450         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 451         minimum => 0,
 452         default => 0.1,
 453     },
 454     cdrom => {
 455         optional => 1,
 456         type => 'string', format => 'pve-qm-ide',
 457         typetext => '<volume>',
 458         description => "This is an alias for option -ide2",
 459     },
 460     cpu => {
 461         optional => 1,
 462         description => "Emulated CPU type.",
 463         type => 'string',
 464         format => $cpu_fmt,
 465     },
 466     parent => get_standard_option('pve-snapshot-name', {
 467         optional => 1,
 468         description => "Parent snapshot name. This is used internally, and should not be modified.",
 469     }),
 470     snaptime => {
 471         optional => 1,
 472         description => "Timestamp for snapshots.",
 473         type => 'integer',
 474         minimum => 0,
 475     },
 476     vmstate => {
 477         optional => 1,
 478         type => 'string', format => 'pve-volume-id',
 479         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 480     },
 481     machine => {
 482         description => "Specific the Qemu machine type.",
 483         type => 'string',
 484         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 485         maxLength => 40,
 486         optional => 1,
 487     },
 488     smbios1 => {
 489         description => "Specify SMBIOS type 1 fields.",
 490         type => 'string', format => 'pve-qm-smbios1',
 491         maxLength => 256,
 492         optional => 1,
 493     },
 494     replicate => get_standard_option('pve-replicate'),
 495     protection => {
 496         optional => 1,
 497         type => 'boolean',
 498         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 499         default => 0,
 500     },
 501     bios => {
 502         optional => 1,
 503         type => 'string',
 504         enum => [ qw(seabios ovmf) ],
 505         description => "Select BIOS implementation.",
 506         default => 'seabios',
 507     },
 508 };
 509
 510 # what about other qemu settings ?
 511 #cpu => 'string',
 512 #machine => 'string',
 513 #fda => 'file',
 514 #fdb => 'file',
 515 #mtdblock => 'file',
 516 #sd => 'file',
 517 #pflash => 'file',
 518 #snapshot => 'bool',
 519 #bootp => 'file',
 520 ##tftp => 'dir',
 521 ##smb => 'dir',
 522 #kernel => 'file',
 523 #append => 'string',
 524 #initrd => 'file',
 525 ##soundhw => 'string',
 526
 527 while (my ($k, $v) = each %$confdesc) {
 528     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 529 }
 530
 531 my $MAX_IDE_DISKS = 4;
 532 my $MAX_SCSI_DISKS = 14;
 533 my $MAX_VIRTIO_DISKS = 16;
 534 my $MAX_SATA_DISKS = 6;
 535 my $MAX_USB_DEVICES = 5;
 536 my $MAX_NETS = 32;
 537 my $MAX_UNUSED_DISKS = 8;
 538 my $MAX_HOSTPCI_DEVICES = 4;
 539 my $MAX_SERIAL_PORTS = 4;
 540 my $MAX_PARALLEL_PORTS = 3;
 541 my $MAX_NUMA = 8;
 542
 543 my $numa_fmt = {
 544     cpus => {
 545         type => "string",
 546         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 547         description => "CPUs accessing this NUMA node.",
 548         format_description => "id[-id];...",
 549     },
 550     memory => {
 551         type => "number",
 552         description => "Amount of memory this NUMA node provides.",
 553         optional => 1,
 554     },
 555     hostnodes => {
 556         type => "string",
 557         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 558         description => "Host NUMA nodes to use.",
 559         format_description => "id[-id];...",
 560         optional => 1,
 561     },
 562     policy => {
 563         type => 'string',
 564         enum => [qw(preferred bind interleave)],
 565         description => "NUMA allocation policy.",
 566         optional => 1,
 567     },
 568 };
 569 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 570 my $numadesc = {
 571     optional => 1,
 572     type => 'string', format => $numa_fmt,
 573     description => "NUMA topology.",
 574 };
 575 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 576
 577 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 578     $confdesc->{"numa$i"} = $numadesc;
 579 }
 580
 581 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 582                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 583                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 584 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 585
 586 my $net_fmt_bridge_descr = <<__EOD__;
 587 Bridge to attach the network device to. The Proxmox VE standard bridge
 588 is called 'vmbr0'.
 589
 590 If you do not specify a bridge, we create a kvm user (NATed) network
 591 device, which provides DHCP and DNS services. The following addresses
 592 are used:
 593
 594  10.0.2.2   Gateway
 595  10.0.2.3   DNS Server
 596  10.0.2.4   SMB Server
 597
 598 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 599 __EOD__
 600
 601 my $net_fmt = {
 602     macaddr => {
 603         type => 'string',
 604         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 605         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 606         format_description => "XX:XX:XX:XX:XX:XX",
 607         optional => 1,
 608     },
 609     model => {
 610         type => 'string',
 611         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 612         enum => $nic_model_list,
 613         default_key => 1,
 614     },
 615     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 616     bridge => {
 617         type => 'string',
 618         description => $net_fmt_bridge_descr,
 619         format_description => 'bridge',
 620         optional => 1,
 621     },
 622     queues => {
 623         type => 'integer',
 624         minimum => 0, maximum => 16,
 625         description => 'Number of packet queues to be used on the device.',
 626         optional => 1,
 627     },
 628     rate => {
 629         type => 'number',
 630         minimum => 0,
 631         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 632         optional => 1,
 633     },
 634     tag => {
 635         type => 'integer',
 636         minimum => 1, maximum => 4094,
 637         description => 'VLAN tag to apply to packets on this interface.',
 638         optional => 1,
 639     },
 640     trunks => {
 641         type => 'string',
 642         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 643         description => 'VLAN trunks to pass through this interface.',
 644         format_description => 'vlanid[;vlanid...]',
 645         optional => 1,
 646     },
 647     firewall => {
 648         type => 'boolean',
 649         description => 'Whether this interface should be protected by the firewall.',
 650         optional => 1,
 651     },
 652     link_down => {
 653         type => 'boolean',
 654         description => 'Whether this interface should be disconnected (like pulling the plug).',
 655         optional => 1,
 656     },
 657 };
 658
 659 my $netdesc = {
 660     optional => 1,
 661     type => 'string', format => $net_fmt,
 662     description => "Specify network devices.",
 663 };
 664
 665 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 666
 667 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 668     $confdesc->{"net$i"} = $netdesc;
 669 }
 670
 671 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 672 sub verify_volume_id_or_qm_path {
 673     my ($volid, $noerr) = @_;
 674
 675     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 676         return $volid;
 677     }
 678
 679     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 680     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 681     if ($@) {
 682         return undef if $noerr;
 683         die $@;
 684     }
 685     return $volid;
 686 }
 687
 688 my $drivename_hash;
 689
 690 my %drivedesc_base = (
 691     volume => { alias => 'file' },
 692     file => {
 693         type => 'string',
 694         format => 'pve-volume-id-or-qm-path',
 695         default_key => 1,
 696         format_description => 'volume',
 697         description => "The drive's backing volume.",
 698     },
 699     media => {
 700         type => 'string',
 701         enum => [qw(cdrom disk)],
 702         description => "The drive's media type.",
 703         default => 'disk',
 704         optional => 1
 705     },
 706     cyls => {
 707         type => 'integer',
 708         description => "Force the drive's physical geometry to have a specific cylinder count.",
 709         optional => 1
 710     },
 711     heads => {
 712         type => 'integer',
 713         description => "Force the drive's physical geometry to have a specific head count.",
 714         optional => 1
 715     },
 716     secs => {
 717         type => 'integer',
 718         description => "Force the drive's physical geometry to have a specific sector count.",
 719         optional => 1
 720     },
 721     trans => {
 722         type => 'string',
 723         enum => [qw(none lba auto)],
 724         description => "Force disk geometry bios translation mode.",
 725         optional => 1,
 726     },
 727     snapshot => {
 728         type => 'boolean',
 729         description => "Whether the drive should be included when making snapshots.",
 730         optional => 1,
 731     },
 732     cache => {
 733         type => 'string',
 734         enum => [qw(none writethrough writeback unsafe directsync)],
 735         description => "The drive's cache mode",
 736         optional => 1,
 737     },
 738     format => get_standard_option('pve-qm-image-format'),
 739     size => {
 740         type => 'string',
 741         format => 'disk-size',
 742         format_description => 'DiskSize',
 743         description => "Disk size. This is purely informational and has no effect.",
 744         optional => 1,
 745     },
 746     backup => {
 747         type => 'boolean',
 748         description => "Whether the drive should be included when making backups.",
 749         optional => 1,
 750     },
 751     replicate => {
 752         type => 'boolean',
 753         description => 'Will include this drive to an storage replical job.',
 754         optional => 1,
 755         default => 1,
 756     },
 757     rerror => {
 758         type => 'string',
 759         enum => [qw(ignore report stop)],
 760         description => 'Read error action.',
 761         optional => 1,
 762     },
 763     werror => {
 764         type => 'string',
 765         enum => [qw(enospc ignore report stop)],
 766         description => 'Write error action.',
 767         optional => 1,
 768     },
 769     aio => {
 770         type => 'string',
 771         enum => [qw(native threads)],
 772         description => 'AIO type to use.',
 773         optional => 1,
 774     },
 775     discard => {
 776         type => 'string',
 777         enum => [qw(ignore on)],
 778         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 779         optional => 1,
 780     },
 781     detect_zeroes => {
 782         type => 'boolean',
 783         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 784         optional => 1,
 785     },
 786     serial => {
 787         type => 'string',
 788         format => 'urlencoded',
 789         format_description => 'serial',
 790         maxLength => 20*3, # *3 since it's %xx url enoded
 791         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 792         optional => 1,
 793     }
 794 );
 795
 796 my %iothread_fmt = ( iothread => {
 797         type => 'boolean',
 798         description => "Whether to use iothreads for this drive",
 799         optional => 1,
 800 });
 801
 802 my %model_fmt = (
 803     model => {
 804         type => 'string',
 805         format => 'urlencoded',
 806         format_description => 'model',
 807         maxLength => 40*3, # *3 since it's %xx url enoded
 808         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 809         optional => 1,
 810     },
 811 );
 812
 813 my %queues_fmt = (
 814     queues => {
 815         type => 'integer',
 816         description => "Number of queues.",
 817         minimum => 2,
 818         optional => 1
 819     }
 820 );
 821
 822 my %scsiblock_fmt = (
 823     scsiblock => {
 824         type => 'boolean',
 825         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 826         optional => 1,
 827         default => 0,
 828     },
 829 );
 830
 831 my $add_throttle_desc = sub {
 832     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 833     my $d = {
 834         type => $type,
 835         format_description => $unit,
 836         description => "Maximum $what in $longunit.",
 837         optional => 1,
 838     };
 839     $d->{minimum} = $minimum if defined($minimum);
 840     $drivedesc_base{$key} = $d;
 841 };
 842 # throughput: (leaky bucket)
 843 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 844 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 845 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 846 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 847 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 848 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 849 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 850 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 851 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 852
 853 # pools: (pool of IO before throttling starts taking effect)
 854 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 855 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 856 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 857 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 858 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 859 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 860
 861 # burst lengths
 862 $add_throttle_desc->('bps_max_length',  'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 863 $add_throttle_desc->('bps_rd_length',   'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 864 $add_throttle_desc->('bps_wr_length',   'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 865 $add_throttle_desc->('iops_max_length', 'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 866 $add_throttle_desc->('iops_rd_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 867 $add_throttle_desc->('iops_wr_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 868
 869 my $ide_fmt = {
 870     %drivedesc_base,
 871     %model_fmt,
 872 };
 873 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 874
 875 my $idedesc = {
 876     optional => 1,
 877     type => 'string', format => $ide_fmt,
 878     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 879 };
 880 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 881
 882 my $scsi_fmt = {
 883     %drivedesc_base,
 884     %iothread_fmt,
 885     %queues_fmt,
 886     %scsiblock_fmt,
 887 };
 888 my $scsidesc = {
 889     optional => 1,
 890     type => 'string', format => $scsi_fmt,
 891     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 892 };
 893 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 894
 895 my $sata_fmt = {
 896     %drivedesc_base,
 897 };
 898 my $satadesc = {
 899     optional => 1,
 900     type => 'string', format => $sata_fmt,
 901     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 902 };
 903 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 904
 905 my $virtio_fmt = {
 906     %drivedesc_base,
 907     %iothread_fmt,
 908 };
 909 my $virtiodesc = {
 910     optional => 1,
 911     type => 'string', format => $virtio_fmt,
 912     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 913 };
 914 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 915
 916 my $alldrive_fmt = {
 917     %drivedesc_base,
 918     %iothread_fmt,
 919     %model_fmt,
 920     %queues_fmt,
 921     %scsiblock_fmt,
 922 };
 923
 924 my $efidisk_fmt = {
 925     volume => { alias => 'file' },
 926     file => {
 927         type => 'string',
 928         format => 'pve-volume-id-or-qm-path',
 929         default_key => 1,
 930         format_description => 'volume',
 931         description => "The drive's backing volume.",
 932     },
 933     format => get_standard_option('pve-qm-image-format'),
 934     size => {
 935         type => 'string',
 936         format => 'disk-size',
 937         format_description => 'DiskSize',
 938         description => "Disk size. This is purely informational and has no effect.",
 939         optional => 1,
 940     },
 941 };
 942
 943 my $efidisk_desc = {
 944     optional => 1,
 945     type => 'string', format => $efidisk_fmt,
 946     description => "Configure a Disk for storing EFI vars",
 947 };
 948
 949 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 950
 951 my $usb_fmt = {
 952     host => {
 953         default_key => 1,
 954         type => 'string', format => 'pve-qm-usb-device',
 955         format_description => 'HOSTUSBDEVICE|spice',
 956         description => <<EODESCR,
 957 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 958
 959  'bus-port(.port)*' (decimal numbers) or
 960  'vendor_id:product_id' (hexadeciaml numbers) or
 961  'spice'
 962
 963 You can use the 'lsusb -t' command to list existing usb devices.
 964
 965 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 966
 967 The value 'spice' can be used to add a usb redirection devices for spice.
 968 EODESCR
 969     },
 970     usb3 => {
 971         optional => 1,
 972         type => 'boolean',
 973         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 974         default => 0,
 975     },
 976 };
 977
 978 my $usbdesc = {
 979     optional => 1,
 980     type => 'string', format => $usb_fmt,
 981     description => "Configure an USB device (n is 0 to 4).",
 982 };
 983 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 984
 985 # NOTE: the match-groups of this regex are used in parse_hostpci
 986 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 987 my $hostpci_fmt = {
 988     host => {
 989         default_key => 1,
 990         type => 'string',
 991         pattern => qr/$PCIRE(;$PCIRE)*/,
 992         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 993         description => <<EODESCR,
 994 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 995 of PCI virtual functions of the host. HOSTPCIID syntax is:
 996
 997 'bus:dev.func' (hexadecimal numbers)
 998
 999 You can us the 'lspci' command to list existing PCI devices.
1000 EODESCR
1001     },
1002     rombar => {
1003         type => 'boolean',
1004         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1005         optional => 1,
1006         default => 1,
1007     },
1008     romfile => {
1009         type => 'string',
1010         pattern => '[^,;]+',
1011         format_description => 'string',
1012         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1013         optional => 1,
1014     },
1015     pcie => {
1016         type => 'boolean',
1017         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1018         optional => 1,
1019         default => 0,
1020     },
1021     'x-vga' => {
1022         type => 'boolean',
1023         description =>  "Enable vfio-vga device support.",
1024         optional => 1,
1025         default => 0,
1026     },
1027 };
1028 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1029
1030 my $hostpcidesc = {
1031         optional => 1,
1032         type => 'string', format => 'pve-qm-hostpci',
1033         description => "Map host PCI devices into guest.",
1034         verbose_description =>  <<EODESCR,
1035 Map host PCI devices into guest.
1036
1037 NOTE: This option allows direct access to host hardware. So it is no longer
1038 possible to migrate such machines - use with special care.
1039
1040 CAUTION: Experimental! User reported problems with this option.
1041 EODESCR
1042 };
1043 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1044
1045 my $serialdesc = {
1046         optional => 1,
1047         type => 'string',
1048         pattern => '(/dev/.+|socket)',
1049         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1050         verbose_description =>  <<EODESCR,
1051 Create a serial device inside the VM (n is 0 to 3), and pass through a
1052 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1053 host side (use 'qm terminal' to open a terminal connection).
1054
1055 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1056
1057 CAUTION: Experimental! User reported problems with this option.
1058 EODESCR
1059 };
1060
1061 my $paralleldesc= {
1062         optional => 1,
1063         type => 'string',
1064         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1065         description =>  "Map host parallel devices (n is 0 to 2).",
1066         verbose_description =>  <<EODESCR,
1067 Map host parallel devices (n is 0 to 2).
1068
1069 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1070
1071 CAUTION: Experimental! User reported problems with this option.
1072 EODESCR
1073 };
1074
1075 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1076     $confdesc->{"parallel$i"} = $paralleldesc;
1077 }
1078
1079 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1080     $confdesc->{"serial$i"} = $serialdesc;
1081 }
1082
1083 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1084     $confdesc->{"hostpci$i"} = $hostpcidesc;
1085 }
1086
1087 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1088     $drivename_hash->{"ide$i"} = 1;
1089     $confdesc->{"ide$i"} = $idedesc;
1090 }
1091
1092 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1093     $drivename_hash->{"sata$i"} = 1;
1094     $confdesc->{"sata$i"} = $satadesc;
1095 }
1096
1097 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1098     $drivename_hash->{"scsi$i"} = 1;
1099     $confdesc->{"scsi$i"} = $scsidesc ;
1100 }
1101
1102 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1103     $drivename_hash->{"virtio$i"} = 1;
1104     $confdesc->{"virtio$i"} = $virtiodesc;
1105 }
1106
1107 $drivename_hash->{efidisk0} = 1;
1108 $confdesc->{efidisk0} = $efidisk_desc;
1109
1110 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1111     $confdesc->{"usb$i"} = $usbdesc;
1112 }
1113
1114 my $unuseddesc = {
1115     optional => 1,
1116     type => 'string', format => 'pve-volume-id',
1117     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1118 };
1119
1120 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1121     $confdesc->{"unused$i"} = $unuseddesc;
1122 }
1123
1124 my $kvm_api_version = 0;
1125
1126 sub kvm_version {
1127
1128     return $kvm_api_version if $kvm_api_version;
1129
1130     my $fh = IO::File->new("</dev/kvm") ||
1131         return 0;
1132
1133     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1134         $kvm_api_version = $v;
1135     }
1136
1137     $fh->close();
1138
1139     return  $kvm_api_version;
1140 }
1141
1142 my $kvm_user_version;
1143
1144 sub kvm_user_version {
1145
1146     return $kvm_user_version if $kvm_user_version;
1147
1148     $kvm_user_version = 'unknown';
1149
1150     my $code = sub {
1151         my $line = shift;
1152         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1153             $kvm_user_version = $2;
1154         }
1155     };
1156
1157     eval { run_command("kvm -version", outfunc => $code); };
1158     warn $@ if $@;
1159
1160     return $kvm_user_version;
1161
1162 }
1163
1164 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1165
1166 sub valid_drive_names {
1167     # order is important - used to autoselect boot disk
1168     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1169             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1170             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1171             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1172             'efidisk0');
1173 }
1174
1175 sub is_valid_drivename {
1176     my $dev = shift;
1177
1178     return defined($drivename_hash->{$dev});
1179 }
1180
1181 sub option_exists {
1182     my $key = shift;
1183     return defined($confdesc->{$key});
1184 }
1185
1186 sub nic_models {
1187     return $nic_model_list;
1188 }
1189
1190 sub os_list_description {
1191
1192     return {
1193         other => 'Other',
1194         wxp => 'Windows XP',
1195         w2k => 'Windows 2000',
1196         w2k3 =>, 'Windows 2003',
1197         w2k8 => 'Windows 2008',
1198         wvista => 'Windows Vista',
1199         win7 => 'Windows 7',
1200         win8 => 'Windows 8/2012',
1201         win10 => 'Windows 10/2016',
1202         l24 => 'Linux 2.4',
1203         l26 => 'Linux 2.6',
1204     };
1205 }
1206
1207 my $cdrom_path;
1208
1209 sub get_cdrom_path {
1210
1211     return  $cdrom_path if $cdrom_path;
1212
1213     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1214     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1215     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1216 }
1217
1218 sub get_iso_path {
1219     my ($storecfg, $vmid, $cdrom) = @_;
1220
1221     if ($cdrom eq 'cdrom') {
1222         return get_cdrom_path();
1223     } elsif ($cdrom eq 'none') {
1224         return '';
1225     } elsif ($cdrom =~ m|^/|) {
1226         return $cdrom;
1227     } else {
1228         return PVE::Storage::path($storecfg, $cdrom);
1229     }
1230 }
1231
1232 # try to convert old style file names to volume IDs
1233 sub filename_to_volume_id {
1234     my ($vmid, $file, $media) = @_;
1235
1236     if (!($file eq 'none' || $file eq 'cdrom' ||
1237           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1238
1239         return undef if $file =~ m|/|;
1240
1241         if ($media && $media eq 'cdrom') {
1242             $file = "local:iso/$file";
1243         } else {
1244             $file = "local:$vmid/$file";
1245         }
1246     }
1247
1248     return $file;
1249 }
1250
1251 sub verify_media_type {
1252     my ($opt, $vtype, $media) = @_;
1253
1254     return if !$media;
1255
1256     my $etype;
1257     if ($media eq 'disk') {
1258         $etype = 'images';
1259     } elsif ($media eq 'cdrom') {
1260         $etype = 'iso';
1261     } else {
1262         die "internal error";
1263     }
1264
1265     return if ($vtype eq $etype);
1266
1267     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1268 }
1269
1270 sub cleanup_drive_path {
1271     my ($opt, $storecfg, $drive) = @_;
1272
1273     # try to convert filesystem paths to volume IDs
1274
1275     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1276         ($drive->{file} !~ m|^/dev/.+|) &&
1277         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1278         ($drive->{file} !~ m/^\d+$/)) {
1279         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1280         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1281         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1282         verify_media_type($opt, $vtype, $drive->{media});
1283         $drive->{file} = $volid;
1284     }
1285
1286     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1287 }
1288
1289 sub parse_hotplug_features {
1290     my ($data) = @_;
1291
1292     my $res = {};
1293
1294     return $res if $data eq '0';
1295
1296     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1297
1298     foreach my $feature (PVE::Tools::split_list($data)) {
1299         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1300             $res->{$1} = 1;
1301         } else {
1302             die "invalid hotplug feature '$feature'\n";
1303         }
1304     }
1305     return $res;
1306 }
1307
1308 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1309 sub pve_verify_hotplug_features {
1310     my ($value, $noerr) = @_;
1311
1312     return $value if parse_hotplug_features($value);
1313
1314     return undef if $noerr;
1315
1316     die "unable to parse hotplug option\n";
1317 }
1318
1319 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1320 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1321 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1322 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1323 #        [,iothread=on][,serial=serial][,model=model]
1324
1325 sub parse_drive {
1326     my ($key, $data) = @_;
1327
1328     my ($interface, $index);
1329
1330     if ($key =~ m/^([^\d]+)(\d+)$/) {
1331         $interface = $1;
1332         $index = $2;
1333     } else {
1334         return undef;
1335     }
1336
1337     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1338                                      : $confdesc->{$key}->{format};
1339     if (!$desc) {
1340         warn "invalid drive key: $key\n";
1341         return undef;
1342     }
1343     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1344     return undef if !$res;
1345     $res->{interface} = $interface;
1346     $res->{index} = $index;
1347
1348     my $error = 0;
1349     foreach my $opt (qw(bps bps_rd bps_wr)) {
1350         if (my $bps = defined(delete $res->{$opt})) {
1351             if (defined($res->{"m$opt"})) {
1352                 warn "both $opt and m$opt specified\n";
1353                 ++$error;
1354                 next;
1355             }
1356             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1357         }
1358     }
1359
1360     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1361     for my $requirement (
1362         [bps_max_length => 'mbps_max'],
1363         [bps_rd_max_length => 'mbps_rd_max'],
1364         [bps_wr_max_length => 'mbps_wr_max'],
1365         [iops_max_length => 'iops_max'],
1366         [iops_rd_max_length => 'iops_rd_max'],
1367         [iops_wr_max_length => 'iops_wr_max']) {
1368         my ($option, $requires) = @$requirement;
1369         if ($res->{$option} && !$res->{$requires}) {
1370             warn "$option requires $requires\n";
1371             ++$error;
1372         }
1373     }
1374
1375     return undef if $error;
1376
1377     return undef if $res->{mbps_rd} && $res->{mbps};
1378     return undef if $res->{mbps_wr} && $res->{mbps};
1379     return undef if $res->{iops_rd} && $res->{iops};
1380     return undef if $res->{iops_wr} && $res->{iops};
1381
1382     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1383         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1384         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1385         return undef if $res->{interface} eq 'virtio';
1386     }
1387
1388     if (my $size = $res->{size}) {
1389         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1390     }
1391
1392     return $res;
1393 }
1394
1395 sub print_drive {
1396     my ($vmid, $drive) = @_;
1397     my $data = { %$drive };
1398     delete $data->{$_} for qw(index interface);
1399     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1400 }
1401
1402 sub scsi_inquiry {
1403     my($fh, $noerr) = @_;
1404
1405     my $SG_IO = 0x2285;
1406     my $SG_GET_VERSION_NUM = 0x2282;
1407
1408     my $versionbuf = "\x00" x 8;
1409     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1410     if (!$ret) {
1411         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1412         return undef;
1413     }
1414     my $version = unpack("I", $versionbuf);
1415     if ($version < 30000) {
1416         die "scsi generic interface too old\n"  if !$noerr;
1417         return undef;
1418     }
1419
1420     my $buf = "\x00" x 36;
1421     my $sensebuf = "\x00" x 8;
1422     my $cmd = pack("C x3 C x1", 0x12, 36);
1423
1424     # see /usr/include/scsi/sg.h
1425     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1426
1427     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1428                       length($sensebuf), 0, length($buf), $buf,
1429                       $cmd, $sensebuf, 6000);
1430
1431     $ret = ioctl($fh, $SG_IO, $packet);
1432     if (!$ret) {
1433         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1434         return undef;
1435     }
1436
1437     my @res = unpack($sg_io_hdr_t, $packet);
1438     if ($res[17] || $res[18]) {
1439         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1440         return undef;
1441     }
1442
1443     my $res = {};
1444     (my $byte0, my $byte1, $res->{vendor},
1445      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1446
1447     $res->{removable} = $byte1 & 128 ? 1 : 0;
1448     $res->{type} = $byte0 & 31;
1449
1450     return $res;
1451 }
1452
1453 sub path_is_scsi {
1454     my ($path) = @_;
1455
1456     my $fh = IO::File->new("+<$path") || return undef;
1457     my $res = scsi_inquiry($fh, 1);
1458     close($fh);
1459
1460     return $res;
1461 }
1462
1463 sub machine_type_is_q35 {
1464     my ($conf) = @_;
1465
1466     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1467 }
1468
1469 sub print_tabletdevice_full {
1470     my ($conf) = @_;
1471
1472     my $q35 = machine_type_is_q35($conf);
1473
1474     # we use uhci for old VMs because tablet driver was buggy in older qemu
1475     my $usbbus = $q35 ? "ehci" : "uhci";
1476
1477     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1478 }
1479
1480 sub print_drivedevice_full {
1481     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1482
1483     my $device = '';
1484     my $maxdev = 0;
1485
1486     if ($drive->{interface} eq 'virtio') {
1487         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1488         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1489         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1490     } elsif ($drive->{interface} eq 'scsi') {
1491
1492         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1493         my $unit = $drive->{index} % $maxdev;
1494         my $devicetype = 'hd';
1495         my $path = '';
1496         if (drive_is_cdrom($drive)) {
1497             $devicetype = 'cd';
1498         } else {
1499             if ($drive->{file} =~ m|^/|) {
1500                 $path = $drive->{file};
1501                 if (my $info = path_is_scsi($path)) {
1502                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1503                         $devicetype = 'block';
1504                     } elsif ($info->{type} == 1) { # tape
1505                         $devicetype = 'generic';
1506                     }
1507                 }
1508             } else {
1509                  $path = PVE::Storage::path($storecfg, $drive->{file});
1510             }
1511
1512             if($path =~ m/^iscsi\:\/\//){
1513                 $devicetype = 'generic';
1514             }
1515         }
1516
1517         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1518             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1519         } else {
1520             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1521         }
1522
1523     } elsif ($drive->{interface} eq 'ide'){
1524         $maxdev = 2;
1525         my $controller = int($drive->{index} / $maxdev);
1526         my $unit = $drive->{index} % $maxdev;
1527         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1528
1529         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1530         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1531             $model = URI::Escape::uri_unescape($model);
1532             $device .= ",model=$model";
1533         }
1534     } elsif ($drive->{interface} eq 'sata'){
1535         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1536         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1537         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1538     } elsif ($drive->{interface} eq 'usb') {
1539         die "implement me";
1540         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1541     } else {
1542         die "unsupported interface type";
1543     }
1544
1545     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1546
1547     return $device;
1548 }
1549
1550 sub get_initiator_name {
1551     my $initiator;
1552
1553     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1554     while (defined(my $line = <$fh>)) {
1555         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1556         $initiator = $1;
1557         last;
1558     }
1559     $fh->close();
1560
1561     return $initiator;
1562 }
1563
1564 sub print_drive_full {
1565     my ($storecfg, $vmid, $drive) = @_;
1566
1567     my $path;
1568     my $volid = $drive->{file};
1569     my $format;
1570
1571     if (drive_is_cdrom($drive)) {
1572         $path = get_iso_path($storecfg, $vmid, $volid);
1573     } else {
1574         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1575         if ($storeid) {
1576             $path = PVE::Storage::path($storecfg, $volid);
1577             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1578             $format = qemu_img_format($scfg, $volname);
1579         } else {
1580             $path = $volid;
1581             $format = "raw";
1582         }
1583    }
1584
1585     my $opts = '';
1586     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1587     foreach my $o (@qemu_drive_options) {
1588         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1589     }
1590     if (my $serial = $drive->{serial}) {
1591         $serial = URI::Escape::uri_unescape($serial);
1592         $opts .= ",serial=$serial";
1593     }
1594
1595     $opts .= ",format=$format" if $format && !$drive->{format};
1596
1597     foreach my $o (qw(bps bps_rd bps_wr)) {
1598         my $v = $drive->{"m$o"};
1599         $opts .= ",$o=" . int($v*1024*1024) if $v;
1600     }
1601
1602     my $cache_direct = 0;
1603
1604     if (my $cache = $drive->{cache}) {
1605         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1606     } elsif (!drive_is_cdrom($drive)) {
1607         $opts .= ",cache=none";
1608         $cache_direct = 1;
1609     }
1610
1611     # aio native works only with O_DIRECT
1612     if (!$drive->{aio}) {
1613         if($cache_direct) {
1614             $opts .= ",aio=native";
1615         } else {
1616             $opts .= ",aio=threads";
1617         }
1618     }
1619
1620     if (!drive_is_cdrom($drive)) {
1621         my $detectzeroes;
1622         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1623             $detectzeroes = 'off';
1624         } elsif ($drive->{discard}) {
1625             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1626         } else {
1627             # This used to be our default with discard not being specified:
1628             $detectzeroes = 'on';
1629         }
1630         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1631     }
1632
1633     my $pathinfo = $path ? "file=$path," : '';
1634
1635     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1636 }
1637
1638 sub print_netdevice_full {
1639     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1640
1641     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1642
1643     my $device = $net->{model};
1644     if ($net->{model} eq 'virtio') {
1645          $device = 'virtio-net-pci';
1646      };
1647
1648     my $pciaddr = print_pci_addr("$netid", $bridges);
1649     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1650     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1651         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1652         my $vectors = $net->{queues} * 2 + 2;
1653         $tmpstr .= ",vectors=$vectors,mq=on";
1654     }
1655     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1656
1657     if ($use_old_bios_files) {
1658         my $romfile;
1659         if ($device eq 'virtio-net-pci') {
1660             $romfile = 'pxe-virtio.rom';
1661         } elsif ($device eq 'e1000') {
1662             $romfile = 'pxe-e1000.rom';
1663         } elsif ($device eq 'ne2k') {
1664             $romfile = 'pxe-ne2k_pci.rom';
1665         } elsif ($device eq 'pcnet') {
1666             $romfile = 'pxe-pcnet.rom';
1667         } elsif ($device eq 'rtl8139') {
1668             $romfile = 'pxe-rtl8139.rom';
1669         }
1670         $tmpstr .= ",romfile=$romfile" if $romfile;
1671     }
1672
1673     return $tmpstr;
1674 }
1675
1676 sub print_netdev_full {
1677     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1678
1679     my $i = '';
1680     if ($netid =~ m/^net(\d+)$/) {
1681         $i = int($1);
1682     }
1683
1684     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1685
1686     my $ifname = "tap${vmid}i$i";
1687
1688     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1689     die "interface name '$ifname' is too long (max 15 character)\n"
1690         if length($ifname) >= 16;
1691
1692     my $vhostparam = '';
1693     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1694
1695     my $vmname = $conf->{name} || "vm$vmid";
1696
1697     my $netdev = "";
1698     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1699
1700     if ($net->{bridge}) {
1701         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1702     } else {
1703         $netdev = "type=user,id=$netid,hostname=$vmname";
1704     }
1705
1706     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1707
1708     return $netdev;
1709 }
1710
1711
1712 sub print_cpu_device {
1713     my ($conf, $id) = @_;
1714
1715     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
1716     my $cpu = $nokvm ? "qemu64" : "kvm64";
1717     if (my $cputype = $conf->{cpu}) {
1718         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1719             or die "Cannot parse cpu description: $cputype\n";
1720         $cpu = $cpuconf->{cputype};
1721     }
1722
1723     my $sockets = 1;
1724     $sockets = $conf->{sockets} if  $conf->{sockets};
1725     my $cores = $conf->{cores} || 1;
1726
1727     my $current_core = ($id - 1) % $cores;
1728     my $current_socket = int(($id - $current_core)/$cores);
1729
1730     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1731 }
1732
1733 sub drive_is_cdrom {
1734     my ($drive) = @_;
1735
1736     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1737
1738 }
1739
1740 sub parse_number_sets {
1741     my ($set) = @_;
1742     my $res = [];
1743     foreach my $part (split(/;/, $set)) {
1744         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1745             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1746             push @$res, [ $1, $2 ];
1747         } else {
1748             die "invalid range: $part\n";
1749         }
1750     }
1751     return $res;
1752 }
1753
1754 sub parse_numa {
1755     my ($data) = @_;
1756
1757     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1758     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1759     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1760     return $res;
1761 }
1762
1763 sub parse_hostpci {
1764     my ($value) = @_;
1765
1766     return undef if !$value;
1767
1768     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1769
1770     my @idlist = split(/;/, $res->{host});
1771     delete $res->{host};
1772     foreach my $id (@idlist) {
1773         if ($id =~ /^$PCIRE$/) {
1774             if (defined($2)) {
1775                 push @{$res->{pciid}}, { id => $1, function => $2 };
1776             } else {
1777                 my $pcidevices = lspci($1);
1778                 $res->{pciid} = $pcidevices->{$1};
1779             }
1780         } else {
1781             # should have been caught by parse_property_string already
1782             die "failed to parse PCI id: $id\n";
1783         }
1784     }
1785     return $res;
1786 }
1787
1788 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1789 sub parse_net {
1790     my ($data) = @_;
1791
1792     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1793     if ($@) {
1794         warn $@;
1795         return undef;
1796     }
1797     if (!defined($res->{macaddr})) {
1798         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1799         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1800     }
1801     return $res;
1802 }
1803
1804 sub print_net {
1805     my $net = shift;
1806
1807     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1808 }
1809
1810 sub add_random_macs {
1811     my ($settings) = @_;
1812
1813     foreach my $opt (keys %$settings) {
1814         next if $opt !~ m/^net(\d+)$/;
1815         my $net = parse_net($settings->{$opt});
1816         next if !$net;
1817         $settings->{$opt} = print_net($net);
1818     }
1819 }
1820
1821 sub vm_is_volid_owner {
1822     my ($storecfg, $vmid, $volid) = @_;
1823
1824     if ($volid !~  m|^/|) {
1825         my ($path, $owner);
1826         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1827         if ($owner && ($owner == $vmid)) {
1828             return 1;
1829         }
1830     }
1831
1832     return undef;
1833 }
1834
1835 sub split_flagged_list {
1836     my $text = shift || '';
1837     $text =~ s/[,;]/ /g;
1838     $text =~ s/^\s+//;
1839     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1840 }
1841
1842 sub join_flagged_list {
1843     my ($how, $lst) = @_;
1844     join $how, map { $lst->{$_} . $_ } keys %$lst;
1845 }
1846
1847 sub vmconfig_delete_pending_option {
1848     my ($conf, $key, $force) = @_;
1849
1850     delete $conf->{pending}->{$key};
1851     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1852     $pending_delete_hash->{$key} = $force ? '!' : '';
1853     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1854 }
1855
1856 sub vmconfig_undelete_pending_option {
1857     my ($conf, $key) = @_;
1858
1859     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1860     delete $pending_delete_hash->{$key};
1861
1862     if (%$pending_delete_hash) {
1863         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1864     } else {
1865         delete $conf->{pending}->{delete};
1866     }
1867 }
1868
1869 sub vmconfig_register_unused_drive {
1870     my ($storecfg, $vmid, $conf, $drive) = @_;
1871
1872     if (!drive_is_cdrom($drive)) {
1873         my $volid = $drive->{file};
1874         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1875             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1876         }
1877     }
1878 }
1879
1880 sub vmconfig_cleanup_pending {
1881     my ($conf) = @_;
1882
1883     # remove pending changes when nothing changed
1884     my $changes;
1885     foreach my $opt (keys %{$conf->{pending}}) {
1886         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1887             $changes = 1;
1888             delete $conf->{pending}->{$opt};
1889         }
1890     }
1891
1892     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1893     my $pending_delete_hash = {};
1894     while (my ($opt, $force) = each %$current_delete_hash) {
1895         if (defined($conf->{$opt})) {
1896             $pending_delete_hash->{$opt} = $force;
1897         } else {
1898             $changes = 1;
1899         }
1900     }
1901
1902     if (%$pending_delete_hash) {
1903         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1904     } else {
1905         delete $conf->{pending}->{delete};
1906     }
1907
1908     return $changes;
1909 }
1910
1911 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1912 my $smbios1_fmt = {
1913     uuid => {
1914         type => 'string',
1915         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1916         format_description => 'UUID',
1917         description => "Set SMBIOS1 UUID.",
1918         optional => 1,
1919     },
1920     version => {
1921         type => 'string',
1922         pattern => '\S+',
1923         format_description => 'string',
1924         description => "Set SMBIOS1 version.",
1925         optional => 1,
1926     },
1927     serial => {
1928         type => 'string',
1929         pattern => '\S+',
1930         format_description => 'string',
1931         description => "Set SMBIOS1 serial number.",
1932         optional => 1,
1933     },
1934     manufacturer => {
1935         type => 'string',
1936         pattern => '\S+',
1937         format_description => 'string',
1938         description => "Set SMBIOS1 manufacturer.",
1939         optional => 1,
1940     },
1941     product => {
1942         type => 'string',
1943         pattern => '\S+',
1944         format_description => 'string',
1945         description => "Set SMBIOS1 product ID.",
1946         optional => 1,
1947     },
1948     sku => {
1949         type => 'string',
1950         pattern => '\S+',
1951         format_description => 'string',
1952         description => "Set SMBIOS1 SKU string.",
1953         optional => 1,
1954     },
1955     family => {
1956         type => 'string',
1957         pattern => '\S+',
1958         format_description => 'string',
1959         description => "Set SMBIOS1 family string.",
1960         optional => 1,
1961     },
1962 };
1963
1964 sub parse_smbios1 {
1965     my ($data) = @_;
1966
1967     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1968     warn $@ if $@;
1969     return $res;
1970 }
1971
1972 sub print_smbios1 {
1973     my ($smbios1) = @_;
1974     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1975 }
1976
1977 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1978
1979 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1980 sub verify_bootdisk {
1981     my ($value, $noerr) = @_;
1982
1983     return $value if is_valid_drivename($value);
1984
1985     return undef if $noerr;
1986
1987     die "invalid boot disk '$value'\n";
1988 }
1989
1990 sub parse_watchdog {
1991     my ($value) = @_;
1992
1993     return undef if !$value;
1994
1995     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1996     warn $@ if $@;
1997     return $res;
1998 }
1999
2000 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2001 sub verify_usb_device {
2002     my ($value, $noerr) = @_;
2003
2004     return $value if parse_usb_device($value);
2005
2006     return undef if $noerr;
2007
2008     die "unable to parse usb device\n";
2009 }
2010
2011 # add JSON properties for create and set function
2012 sub json_config_properties {
2013     my $prop = shift;
2014
2015     foreach my $opt (keys %$confdesc) {
2016         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2017         $prop->{$opt} = $confdesc->{$opt};
2018     }
2019
2020     return $prop;
2021 }
2022
2023 sub check_type {
2024     my ($key, $value) = @_;
2025
2026     die "unknown setting '$key'\n" if !$confdesc->{$key};
2027
2028     my $type = $confdesc->{$key}->{type};
2029
2030     if (!defined($value)) {
2031         die "got undefined value\n";
2032     }
2033
2034     if ($value =~ m/[\n\r]/) {
2035         die "property contains a line feed\n";
2036     }
2037
2038     if ($type eq 'boolean') {
2039         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2040         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2041         die "type check ('boolean') failed - got '$value'\n";
2042     } elsif ($type eq 'integer') {
2043         return int($1) if $value =~ m/^(\d+)$/;
2044         die "type check ('integer') failed - got '$value'\n";
2045     } elsif ($type eq 'number') {
2046         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2047         die "type check ('number') failed - got '$value'\n";
2048     } elsif ($type eq 'string') {
2049         if (my $fmt = $confdesc->{$key}->{format}) {
2050             PVE::JSONSchema::check_format($fmt, $value);
2051             return $value;
2052         }
2053         $value =~ s/^\"(.*)\"$/$1/;
2054         return $value;
2055     } else {
2056         die "internal error"
2057     }
2058 }
2059
2060 sub check_iommu_support{
2061     #fixme : need to check IOMMU support
2062     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2063
2064     my $iommu=1;
2065     return $iommu;
2066
2067 }
2068
2069 sub touch_config {
2070     my ($vmid) = @_;
2071
2072     my $conf = PVE::QemuConfig->config_file($vmid);
2073     utime undef, undef, $conf;
2074 }
2075
2076 sub destroy_vm {
2077     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2078
2079     my $conffile = PVE::QemuConfig->config_file($vmid);
2080
2081     my $conf = PVE::QemuConfig->load_config($vmid);
2082
2083     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2084
2085     # only remove disks owned by this VM
2086     foreach_drive($conf, sub {
2087         my ($ds, $drive) = @_;
2088
2089         return if drive_is_cdrom($drive);
2090
2091         my $volid = $drive->{file};
2092
2093         return if !$volid || $volid =~ m|^/|;
2094
2095         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2096         return if !$path || !$owner || ($owner != $vmid);
2097
2098         eval {
2099             PVE::Storage::vdisk_free($storecfg, $volid);
2100         };
2101         warn "Could not remove disk '$volid', check manually: $@" if $@;
2102
2103     });
2104
2105     if ($keep_empty_config) {
2106         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2107     } else {
2108         unlink $conffile;
2109     }
2110
2111     # also remove unused disk
2112     eval {
2113         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2114
2115         eval {
2116             PVE::Storage::foreach_volid($dl, sub {
2117                 my ($volid, $sid, $volname, $d) = @_;
2118                 PVE::Storage::vdisk_free($storecfg, $volid);
2119             });
2120         };
2121         warn $@ if $@;
2122
2123     };
2124     warn $@ if $@;
2125 }
2126
2127 sub parse_vm_config {
2128     my ($filename, $raw) = @_;
2129
2130     return undef if !defined($raw);
2131
2132     my $res = {
2133         digest => Digest::SHA::sha1_hex($raw),
2134         snapshots => {},
2135         pending => {},
2136     };
2137
2138     $filename =~ m|/qemu-server/(\d+)\.conf$|
2139         || die "got strange filename '$filename'";
2140
2141     my $vmid = $1;
2142
2143     my $conf = $res;
2144     my $descr;
2145     my $section = '';
2146
2147     my @lines = split(/\n/, $raw);
2148     foreach my $line (@lines) {
2149         next if $line =~ m/^\s*$/;
2150
2151         if ($line =~ m/^\[PENDING\]\s*$/i) {
2152             $section = 'pending';
2153             if (defined($descr)) {
2154                 $descr =~ s/\s+$//;
2155                 $conf->{description} = $descr;
2156             }
2157             $descr = undef;
2158             $conf = $res->{$section} = {};
2159             next;
2160
2161         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2162             $section = $1;
2163             if (defined($descr)) {
2164                 $descr =~ s/\s+$//;
2165                 $conf->{description} = $descr;
2166             }
2167             $descr = undef;
2168             $conf = $res->{snapshots}->{$section} = {};
2169             next;
2170         }
2171
2172         if ($line =~ m/^\#(.*)\s*$/) {
2173             $descr = '' if !defined($descr);
2174             $descr .= PVE::Tools::decode_text($1) . "\n";
2175             next;
2176         }
2177
2178         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2179             $descr = '' if !defined($descr);
2180             $descr .= PVE::Tools::decode_text($2);
2181         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2182             $conf->{snapstate} = $1;
2183         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2184             my $key = $1;
2185             my $value = $2;
2186             $conf->{$key} = $value;
2187         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2188             my $value = $1;
2189             if ($section eq 'pending') {
2190                 $conf->{delete} = $value; # we parse this later
2191             } else {
2192                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2193             }
2194         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2195             my $key = $1;
2196             my $value = $2;
2197             eval { $value = check_type($key, $value); };
2198             if ($@) {
2199                 warn "vm $vmid - unable to parse value of '$key' - $@";
2200             } else {
2201                 $key = 'ide2' if $key eq 'cdrom';
2202                 my $fmt = $confdesc->{$key}->{format};
2203                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2204                     my $v = parse_drive($key, $value);
2205                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2206                         $v->{file} = $volid;
2207                         $value = print_drive($vmid, $v);
2208                     } else {
2209                         warn "vm $vmid - unable to parse value of '$key'\n";
2210                         next;
2211                     }
2212                 }
2213
2214                 $conf->{$key} = $value;
2215             }
2216         }
2217     }
2218
2219     if (defined($descr)) {
2220         $descr =~ s/\s+$//;
2221         $conf->{description} = $descr;
2222     }
2223     delete $res->{snapstate}; # just to be sure
2224
2225     return $res;
2226 }
2227
2228 sub write_vm_config {
2229     my ($filename, $conf) = @_;
2230
2231     delete $conf->{snapstate}; # just to be sure
2232
2233     if ($conf->{cdrom}) {
2234         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2235         $conf->{ide2} = $conf->{cdrom};
2236         delete $conf->{cdrom};
2237     }
2238
2239     # we do not use 'smp' any longer
2240     if ($conf->{sockets}) {
2241         delete $conf->{smp};
2242     } elsif ($conf->{smp}) {
2243         $conf->{sockets} = $conf->{smp};
2244         delete $conf->{cores};
2245         delete $conf->{smp};
2246     }
2247
2248     my $used_volids = {};
2249
2250     my $cleanup_config = sub {
2251         my ($cref, $pending, $snapname) = @_;
2252
2253         foreach my $key (keys %$cref) {
2254             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2255                 $key eq 'snapstate' || $key eq 'pending';
2256             my $value = $cref->{$key};
2257             if ($key eq 'delete') {
2258                 die "propertry 'delete' is only allowed in [PENDING]\n"
2259                     if !$pending;
2260                 # fixme: check syntax?
2261                 next;
2262             }
2263             eval { $value = check_type($key, $value); };
2264             die "unable to parse value of '$key' - $@" if $@;
2265
2266             $cref->{$key} = $value;
2267
2268             if (!$snapname && is_valid_drivename($key)) {
2269                 my $drive = parse_drive($key, $value);
2270                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2271             }
2272         }
2273     };
2274
2275     &$cleanup_config($conf);
2276
2277     &$cleanup_config($conf->{pending}, 1);
2278
2279     foreach my $snapname (keys %{$conf->{snapshots}}) {
2280         die "internal error" if $snapname eq 'pending';
2281         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2282     }
2283
2284     # remove 'unusedX' settings if we re-add a volume
2285     foreach my $key (keys %$conf) {
2286         my $value = $conf->{$key};
2287         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2288             delete $conf->{$key};
2289         }
2290     }
2291
2292     my $generate_raw_config = sub {
2293         my ($conf, $pending) = @_;
2294
2295         my $raw = '';
2296
2297         # add description as comment to top of file
2298         if (defined(my $descr = $conf->{description})) {
2299             if ($descr) {
2300                 foreach my $cl (split(/\n/, $descr)) {
2301                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2302                 }
2303             } else {
2304                 $raw .= "#\n" if $pending;
2305             }
2306         }
2307
2308         foreach my $key (sort keys %$conf) {
2309             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2310             $raw .= "$key: $conf->{$key}\n";
2311         }
2312         return $raw;
2313     };
2314
2315     my $raw = &$generate_raw_config($conf);
2316
2317     if (scalar(keys %{$conf->{pending}})){
2318         $raw .= "\n[PENDING]\n";
2319         $raw .= &$generate_raw_config($conf->{pending}, 1);
2320     }
2321
2322     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2323         $raw .= "\n[$snapname]\n";
2324         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2325     }
2326
2327     return $raw;
2328 }
2329
2330 sub load_defaults {
2331
2332     my $res = {};
2333
2334     # we use static defaults from our JSON schema configuration
2335     foreach my $key (keys %$confdesc) {
2336         if (defined(my $default = $confdesc->{$key}->{default})) {
2337             $res->{$key} = $default;
2338         }
2339     }
2340
2341     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2342     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2343
2344     return $res;
2345 }
2346
2347 sub config_list {
2348     my $vmlist = PVE::Cluster::get_vmlist();
2349     my $res = {};
2350     return $res if !$vmlist || !$vmlist->{ids};
2351     my $ids = $vmlist->{ids};
2352
2353     foreach my $vmid (keys %$ids) {
2354         my $d = $ids->{$vmid};
2355         next if !$d->{node} || $d->{node} ne $nodename;
2356         next if !$d->{type} || $d->{type} ne 'qemu';
2357         $res->{$vmid}->{exists} = 1;
2358     }
2359     return $res;
2360 }
2361
2362 # test if VM uses local resources (to prevent migration)
2363 sub check_local_resources {
2364     my ($conf, $noerr) = @_;
2365
2366     my $loc_res = 0;
2367
2368     $loc_res = 1 if $conf->{hostusb}; # old syntax
2369     $loc_res = 1 if $conf->{hostpci}; # old syntax
2370
2371     foreach my $k (keys %$conf) {
2372         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2373         # sockets are safe: they will recreated be on the target side post-migrate
2374         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2375         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2376     }
2377
2378     die "VM uses local resources\n" if $loc_res && !$noerr;
2379
2380     return $loc_res;
2381 }
2382
2383 # check if used storages are available on all nodes (use by migrate)
2384 sub check_storage_availability {
2385     my ($storecfg, $conf, $node) = @_;
2386
2387     foreach_drive($conf, sub {
2388         my ($ds, $drive) = @_;
2389
2390         my $volid = $drive->{file};
2391         return if !$volid;
2392
2393         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2394         return if !$sid;
2395
2396         # check if storage is available on both nodes
2397         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2398         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2399    });
2400 }
2401
2402 # list nodes where all VM images are available (used by has_feature API)
2403 sub shared_nodes {
2404     my ($conf, $storecfg) = @_;
2405
2406     my $nodelist = PVE::Cluster::get_nodelist();
2407     my $nodehash = { map { $_ => 1 } @$nodelist };
2408     my $nodename = PVE::INotify::nodename();
2409
2410     foreach_drive($conf, sub {
2411         my ($ds, $drive) = @_;
2412
2413         my $volid = $drive->{file};
2414         return if !$volid;
2415
2416         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2417         if ($storeid) {
2418             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2419             if ($scfg->{disable}) {
2420                 $nodehash = {};
2421             } elsif (my $avail = $scfg->{nodes}) {
2422                 foreach my $node (keys %$nodehash) {
2423                     delete $nodehash->{$node} if !$avail->{$node};
2424                 }
2425             } elsif (!$scfg->{shared}) {
2426                 foreach my $node (keys %$nodehash) {
2427                     delete $nodehash->{$node} if $node ne $nodename
2428                 }
2429             }
2430         }
2431     });
2432
2433     return $nodehash
2434 }
2435
2436 sub check_cmdline {
2437     my ($pidfile, $pid) = @_;
2438
2439     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2440     if (defined($fh)) {
2441         my $line = <$fh>;
2442         $fh->close;
2443         return undef if !$line;
2444         my @param = split(/\0/, $line);
2445
2446         my $cmd = $param[0];
2447         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2448
2449         for (my $i = 0; $i < scalar (@param); $i++) {
2450             my $p = $param[$i];
2451             next if !$p;
2452             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2453                 my $p = $param[$i+1];
2454                 return 1 if $p && ($p eq $pidfile);
2455                 return undef;
2456             }
2457         }
2458     }
2459     return undef;
2460 }
2461
2462 sub check_running {
2463     my ($vmid, $nocheck, $node) = @_;
2464
2465     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2466
2467     die "unable to find configuration file for VM $vmid - no such machine\n"
2468         if !$nocheck && ! -f $filename;
2469
2470     my $pidfile = pidfile_name($vmid);
2471
2472     if (my $fd = IO::File->new("<$pidfile")) {
2473         my $st = stat($fd);
2474         my $line = <$fd>;
2475         close($fd);
2476
2477         my $mtime = $st->mtime;
2478         if ($mtime > time()) {
2479             warn "file '$filename' modified in future\n";
2480         }
2481
2482         if ($line =~ m/^(\d+)$/) {
2483             my $pid = $1;
2484             if (check_cmdline($pidfile, $pid)) {
2485                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2486                     return $pid;
2487                 }
2488             }
2489         }
2490     }
2491
2492     return undef;
2493 }
2494
2495 sub vzlist {
2496
2497     my $vzlist = config_list();
2498
2499     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2500
2501     while (defined(my $de = $fd->read)) {
2502         next if $de !~ m/^(\d+)\.pid$/;
2503         my $vmid = $1;
2504         next if !defined($vzlist->{$vmid});
2505         if (my $pid = check_running($vmid)) {
2506             $vzlist->{$vmid}->{pid} = $pid;
2507         }
2508     }
2509
2510     return $vzlist;
2511 }
2512
2513 sub disksize {
2514     my ($storecfg, $conf) = @_;
2515
2516     my $bootdisk = $conf->{bootdisk};
2517     return undef if !$bootdisk;
2518     return undef if !is_valid_drivename($bootdisk);
2519
2520     return undef if !$conf->{$bootdisk};
2521
2522     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2523     return undef if !defined($drive);
2524
2525     return undef if drive_is_cdrom($drive);
2526
2527     my $volid = $drive->{file};
2528     return undef if !$volid;
2529
2530     return $drive->{size};
2531 }
2532
2533 my $last_proc_pid_stat;
2534
2535 # get VM status information
2536 # This must be fast and should not block ($full == false)
2537 # We only query KVM using QMP if $full == true (this can be slow)
2538 sub vmstatus {
2539     my ($opt_vmid, $full) = @_;
2540
2541     my $res = {};
2542
2543     my $storecfg = PVE::Storage::config();
2544
2545     my $list = vzlist();
2546     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2547
2548     my $cpucount = $cpuinfo->{cpus} || 1;
2549
2550     foreach my $vmid (keys %$list) {
2551         next if $opt_vmid && ($vmid ne $opt_vmid);
2552
2553         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2554         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2555
2556         my $d = {};
2557         $d->{pid} = $list->{$vmid}->{pid};
2558
2559         # fixme: better status?
2560         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2561
2562         my $size = disksize($storecfg, $conf);
2563         if (defined($size)) {
2564             $d->{disk} = 0; # no info available
2565             $d->{maxdisk} = $size;
2566         } else {
2567             $d->{disk} = 0;
2568             $d->{maxdisk} = 0;
2569         }
2570
2571         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2572         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2573         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2574
2575         $d->{name} = $conf->{name} || "VM $vmid";
2576         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2577
2578         if ($conf->{balloon}) {
2579             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2580             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2581         }
2582
2583         $d->{uptime} = 0;
2584         $d->{cpu} = 0;
2585         $d->{mem} = 0;
2586
2587         $d->{netout} = 0;
2588         $d->{netin} = 0;
2589
2590         $d->{diskread} = 0;
2591         $d->{diskwrite} = 0;
2592
2593         $d->{template} = PVE::QemuConfig->is_template($conf);
2594
2595         $res->{$vmid} = $d;
2596     }
2597
2598     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2599     foreach my $dev (keys %$netdev) {
2600         next if $dev !~ m/^tap([1-9]\d*)i/;
2601         my $vmid = $1;
2602         my $d = $res->{$vmid};
2603         next if !$d;
2604
2605         $d->{netout} += $netdev->{$dev}->{receive};
2606         $d->{netin} += $netdev->{$dev}->{transmit};
2607
2608         if ($full) {
2609             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2610             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2611         }
2612
2613     }
2614
2615     my $ctime = gettimeofday;
2616
2617     foreach my $vmid (keys %$list) {
2618
2619         my $d = $res->{$vmid};
2620         my $pid = $d->{pid};
2621         next if !$pid;
2622
2623         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2624         next if !$pstat; # not running
2625
2626         my $used = $pstat->{utime} + $pstat->{stime};
2627
2628         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2629
2630         if ($pstat->{vsize}) {
2631             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2632         }
2633
2634         my $old = $last_proc_pid_stat->{$pid};
2635         if (!$old) {
2636             $last_proc_pid_stat->{$pid} = {
2637                 time => $ctime,
2638                 used => $used,
2639                 cpu => 0,
2640             };
2641             next;
2642         }
2643
2644         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2645
2646         if ($dtime > 1000) {
2647             my $dutime = $used -  $old->{used};
2648
2649             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2650             $last_proc_pid_stat->{$pid} = {
2651                 time => $ctime,
2652                 used => $used,
2653                 cpu => $d->{cpu},
2654             };
2655         } else {
2656             $d->{cpu} = $old->{cpu};
2657         }
2658     }
2659
2660     return $res if !$full;
2661
2662     my $qmpclient = PVE::QMPClient->new();
2663
2664     my $ballooncb = sub {
2665         my ($vmid, $resp) = @_;
2666
2667         my $info = $resp->{'return'};
2668         return if !$info->{max_mem};
2669
2670         my $d = $res->{$vmid};
2671
2672         # use memory assigned to VM
2673         $d->{maxmem} = $info->{max_mem};
2674         $d->{balloon} = $info->{actual};
2675
2676         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2677             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2678             $d->{freemem} = $info->{free_mem};
2679         }
2680
2681         $d->{ballooninfo} = $info;
2682     };
2683
2684     my $blockstatscb = sub {
2685         my ($vmid, $resp) = @_;
2686         my $data = $resp->{'return'} || [];
2687         my $totalrdbytes = 0;
2688         my $totalwrbytes = 0;
2689
2690         for my $blockstat (@$data) {
2691             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2692             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2693
2694             $blockstat->{device} =~ s/drive-//;
2695             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2696         }
2697         $res->{$vmid}->{diskread} = $totalrdbytes;
2698         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2699     };
2700
2701     my $statuscb = sub {
2702         my ($vmid, $resp) = @_;
2703
2704         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2705         # this fails if ballon driver is not loaded, so this must be
2706         # the last commnand (following command are aborted if this fails).
2707         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2708
2709         my $status = 'unknown';
2710         if (!defined($status = $resp->{'return'}->{status})) {
2711             warn "unable to get VM status\n";
2712             return;
2713         }
2714
2715         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2716     };
2717
2718     foreach my $vmid (keys %$list) {
2719         next if $opt_vmid && ($vmid ne $opt_vmid);
2720         next if !$res->{$vmid}->{pid}; # not running
2721         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2722     }
2723
2724     $qmpclient->queue_execute(undef, 2);
2725
2726     foreach my $vmid (keys %$list) {
2727         next if $opt_vmid && ($vmid ne $opt_vmid);
2728         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2729     }
2730
2731     return $res;
2732 }
2733
2734 sub foreach_drive {
2735     my ($conf, $func, @param) = @_;
2736
2737     foreach my $ds (valid_drive_names()) {
2738         next if !defined($conf->{$ds});
2739
2740         my $drive = parse_drive($ds, $conf->{$ds});
2741         next if !$drive;
2742
2743         &$func($ds, $drive, @param);
2744     }
2745 }
2746
2747 sub foreach_volid {
2748     my ($conf, $func, @param) = @_;
2749
2750     my $volhash = {};
2751
2752     my $test_volid = sub {
2753         my ($volid, $is_cdrom) = @_;
2754
2755         return if !$volid;
2756
2757         $volhash->{$volid} = $is_cdrom || 0;
2758     };
2759
2760     foreach_drive($conf, sub {
2761         my ($ds, $drive) = @_;
2762         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2763     });
2764
2765     foreach my $snapname (keys %{$conf->{snapshots}}) {
2766         my $snap = $conf->{snapshots}->{$snapname};
2767         &$test_volid($snap->{vmstate}, 0);
2768         foreach_drive($snap, sub {
2769             my ($ds, $drive) = @_;
2770             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2771         });
2772     }
2773
2774     foreach my $volid (keys %$volhash) {
2775         &$func($volid, $volhash->{$volid}, @param);
2776     }
2777 }
2778
2779 sub get_replicatable_volumes {
2780     my ($storecfg, $conf, $noerr) = @_;
2781
2782     my $volhash = {};
2783
2784     my $test_volid = sub {
2785         my ($volid, $drive) = @_;
2786
2787         return if !$volid;
2788
2789         return if drive_is_cdrom($drive);
2790
2791         return if defined($drive->{replicate}) && !$drive->{replicate};
2792
2793         if (!PVE::Storage::volume_has_feature($storecfg, 'replicate', $volid)) {
2794             return if $noerr;
2795             die "missing replicate feature on volume '$volid'\n";
2796         }
2797
2798         $volhash->{$volid} = 1;
2799     };
2800
2801     foreach_drive($conf, sub {
2802         my ($ds, $drive) = @_;
2803         $test_volid->($drive->{file}, $drive);
2804     });
2805
2806     foreach my $snapname (keys %{$conf->{snapshots}}) {
2807         my $snap = $conf->{snapshots}->{$snapname};
2808         # fixme: what about $snap->{vmstate}
2809         foreach_drive($snap, sub {
2810             my ($ds, $drive) = @_;
2811             $test_volid->($drive->{file}, $drive);
2812         });
2813     }
2814
2815     return $volhash;
2816 }
2817
2818 sub vga_conf_has_spice {
2819     my ($vga) = @_;
2820
2821     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2822
2823     return $1 || 1;
2824 }
2825
2826 sub config_to_command {
2827     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2828
2829     my $cmd = [];
2830     my $globalFlags = [];
2831     my $machineFlags = [];
2832     my $rtcFlags = [];
2833     my $cpuFlags = [];
2834     my $devices = [];
2835     my $pciaddr = '';
2836     my $bridges = {};
2837     my $kvmver = kvm_user_version();
2838     my $vernum = 0; # unknown
2839     my $ostype = $conf->{ostype};
2840     my $winversion = windows_version($ostype);
2841
2842     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2843         $vernum = $1*1000000+$2*1000;
2844     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2845         $vernum = $1*1000000+$2*1000+$3;
2846     }
2847
2848     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2849
2850     my $have_ovz = -f '/proc/vz/vestat';
2851
2852     my $q35 = machine_type_is_q35($conf);
2853     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2854     my $machine_type = $forcemachine || $conf->{machine};
2855     my $use_old_bios_files = undef;
2856     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2857
2858     my $cpuunits = defined($conf->{cpuunits}) ?
2859             $conf->{cpuunits} : $defaults->{cpuunits};
2860
2861     push @$cmd, '/usr/bin/kvm';
2862
2863     push @$cmd, '-id', $vmid;
2864
2865     my $use_virtio = 0;
2866
2867     my $qmpsocket = qmp_socket($vmid);
2868     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2869     push @$cmd, '-mon', "chardev=qmp,mode=control";
2870
2871
2872     push @$cmd, '-pidfile' , pidfile_name($vmid);
2873
2874     push @$cmd, '-daemonize';
2875
2876     if ($conf->{smbios1}) {
2877         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2878     }
2879
2880     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2881         my $ovmfbase;
2882
2883         # prefer the OVMF_CODE variant
2884         if (-f $OVMF_CODE) {
2885             $ovmfbase = $OVMF_CODE;
2886         } elsif (-f $OVMF_IMG) {
2887             $ovmfbase = $OVMF_IMG;
2888         }
2889
2890         die "no uefi base img found\n" if !$ovmfbase;
2891         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmfbase";
2892
2893         if (defined($conf->{efidisk0}) && ($ovmfbase eq $OVMF_CODE)) {
2894             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $conf->{efidisk0});
2895             my $format = $d->{format} // 'raw';
2896             my $path;
2897             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2898             if ($storeid) {
2899                 $path = PVE::Storage::path($storecfg, $d->{file});
2900                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2901                 $format = qemu_img_format($scfg, $volname);
2902             } else {
2903                 $path = $d->{file};
2904                 $format = "raw";
2905             }
2906             push @$cmd, '-drive', "if=pflash,unit=1,id=drive-efidisk0,format=$format,file=$path";
2907         } elsif ($ovmfbase eq $OVMF_CODE) {
2908             warn "using uefi without permanent efivars disk\n";
2909             my $ovmfvar_dst = "/tmp/$vmid-ovmf.fd";
2910             PVE::Tools::file_copy($OVMF_VARS, $ovmfvar_dst, 256*1024);
2911             push @$cmd, '-drive', "if=pflash,unit=1,format=raw,file=$ovmfvar_dst";
2912         } else {
2913             # if the base img is not OVMF_CODE, we do not have to bother
2914             # to create/use a vars image, since it will not be used anyway
2915             # this can only happen if someone manually deletes the OVMF_CODE image
2916             # or has an old pve-qemu-kvm version installed.
2917             # both should not happen, but we ignore it here
2918         }
2919     }
2920
2921
2922     # add usb controllers
2923     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2924     push @$devices, @usbcontrollers if @usbcontrollers;
2925     my $vga = $conf->{vga};
2926
2927     my $qxlnum = vga_conf_has_spice($vga);
2928     $vga = 'qxl' if $qxlnum;
2929
2930     if (!$vga) {
2931         $vga = $winversion >= 6 ? 'std' : 'cirrus';
2932     }
2933
2934     # enable absolute mouse coordinates (needed by vnc)
2935     my $tablet;
2936     if (defined($conf->{tablet})) {
2937         $tablet = $conf->{tablet};
2938     } else {
2939         $tablet = $defaults->{tablet};
2940         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2941         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2942     }
2943
2944     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2945
2946     my $kvm_off = 0;
2947     my $gpu_passthrough;
2948
2949     # host pci devices
2950     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2951         my $d = parse_hostpci($conf->{"hostpci$i"});
2952         next if !$d;
2953
2954         my $pcie = $d->{pcie};
2955         if($pcie){
2956             die "q35 machine model is not enabled" if !$q35;
2957             $pciaddr = print_pcie_addr("hostpci$i");
2958         }else{
2959             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2960         }
2961
2962         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2963         my $romfile = $d->{romfile};
2964
2965         my $xvga = '';
2966         if ($d->{'x-vga'}) {
2967             $xvga = ',x-vga=on';
2968             $kvm_off = 1;
2969             $vga = 'none';
2970             $gpu_passthrough = 1;
2971
2972             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2973                 $xvga = "";
2974             }
2975         }
2976         my $pcidevices = $d->{pciid};
2977         my $multifunction = 1 if @$pcidevices > 1;
2978
2979         my $j=0;
2980         foreach my $pcidevice (@$pcidevices) {
2981
2982             my $id = "hostpci$i";
2983             $id .= ".$j" if $multifunction;
2984             my $addr = $pciaddr;
2985             $addr .= ".$j" if $multifunction;
2986             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2987
2988             if($j == 0){
2989                 $devicestr .= "$rombar$xvga";
2990                 $devicestr .= ",multifunction=on" if $multifunction;
2991                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
2992             }
2993
2994             push @$devices, '-device', $devicestr;
2995             $j++;
2996         }
2997     }
2998
2999     # usb devices
3000     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3001     push @$devices, @usbdevices if @usbdevices;
3002     # serial devices
3003     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3004         if (my $path = $conf->{"serial$i"}) {
3005             if ($path eq 'socket') {
3006                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3007                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3008                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3009             } else {
3010                 die "no such serial device\n" if ! -c $path;
3011                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3012                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3013             }
3014         }
3015     }
3016
3017     # parallel devices
3018     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3019         if (my $path = $conf->{"parallel$i"}) {
3020             die "no such parallel device\n" if ! -c $path;
3021             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3022             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3023             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3024         }
3025     }
3026
3027     my $vmname = $conf->{name} || "vm$vmid";
3028
3029     push @$cmd, '-name', $vmname;
3030
3031     my $sockets = 1;
3032     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3033     $sockets = $conf->{sockets} if  $conf->{sockets};
3034
3035     my $cores = $conf->{cores} || 1;
3036
3037     my $maxcpus = $sockets * $cores;
3038
3039     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3040
3041     my $allowed_vcpus = $cpuinfo->{cpus};
3042
3043     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3044         if ($allowed_vcpus < $maxcpus);
3045
3046     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3047
3048         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3049         for (my $i = 2; $i <= $vcpus; $i++)  {
3050             my $cpustr = print_cpu_device($conf,$i);
3051             push @$cmd, '-device', $cpustr;
3052         }
3053
3054     } else {
3055
3056         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3057     }
3058     push @$cmd, '-nodefaults';
3059
3060     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3061
3062     my $bootindex_hash = {};
3063     my $i = 1;
3064     foreach my $o (split(//, $bootorder)) {
3065         $bootindex_hash->{$o} = $i*100;
3066         $i++;
3067     }
3068
3069     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3070
3071     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3072
3073     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3074
3075     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3076
3077     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3078         my $socket = vnc_socket($vmid);
3079         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3080     } else {
3081         push @$cmd, '-nographic';
3082     }
3083
3084     # time drift fix
3085     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3086
3087     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3088     my $useLocaltime = $conf->{localtime};
3089
3090     if ($winversion >= 5) { # windows
3091         $useLocaltime = 1 if !defined($conf->{localtime});
3092
3093         # use time drift fix when acpi is enabled
3094         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3095             $tdf = 1 if !defined($conf->{tdf});
3096         }
3097     }
3098
3099     if ($winversion >= 6) {
3100         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3101         push @$cmd, '-no-hpet';
3102     }
3103
3104     push @$rtcFlags, 'driftfix=slew' if $tdf;
3105
3106     if ($nokvm) {
3107         push @$machineFlags, 'accel=tcg';
3108     } else {
3109         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3110     }
3111
3112     if ($machine_type) {
3113         push @$machineFlags, "type=${machine_type}";
3114     }
3115
3116     if ($conf->{startdate}) {
3117         push @$rtcFlags, "base=$conf->{startdate}";
3118     } elsif ($useLocaltime) {
3119         push @$rtcFlags, 'base=localtime';
3120     }
3121
3122     my $cpu = $nokvm ? "qemu64" : "kvm64";
3123     if (my $cputype = $conf->{cpu}) {
3124         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3125             or die "Cannot parse cpu description: $cputype\n";
3126         $cpu = $cpuconf->{cputype};
3127         $kvm_off = 1 if $cpuconf->{hidden};
3128     }
3129
3130     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3131
3132     push @$cpuFlags , '-x2apic'
3133         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3134
3135     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3136
3137     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3138
3139     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3140
3141         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3142         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3143     }
3144
3145     add_hyperv_enlighments($cpuFlags, $winversion, $machine_type, $kvmver, $nokvm, $conf->{bios}, $gpu_passthrough);
3146
3147     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3148
3149     push @$cpuFlags, 'kvm=off' if $kvm_off;
3150
3151     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3152         die "internal error"; # should not happen
3153
3154     push @$cpuFlags, "vendor=${cpu_vendor}"
3155         if $cpu_vendor ne 'default';
3156
3157     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3158
3159     push @$cmd, '-cpu', $cpu;
3160
3161     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3162
3163     push @$cmd, '-S' if $conf->{freeze};
3164
3165     # set keyboard layout
3166     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3167     push @$cmd, '-k', $kb if $kb;
3168
3169     # enable sound
3170     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3171     #push @$cmd, '-soundhw', 'es1370';
3172     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3173
3174     if($conf->{agent}) {
3175         my $qgasocket = qmp_socket($vmid, 1);
3176         my $pciaddr = print_pci_addr("qga0", $bridges);
3177         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3178         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3179         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3180     }
3181
3182     my $spice_port;
3183
3184     if ($qxlnum) {
3185         if ($qxlnum > 1) {
3186             if ($winversion){
3187                 for(my $i = 1; $i < $qxlnum; $i++){
3188                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3189                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3190                 }
3191             } else {
3192                 # assume other OS works like Linux
3193                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3194                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3195             }
3196         }
3197
3198         my $pciaddr = print_pci_addr("spice", $bridges);
3199
3200         my $nodename = PVE::INotify::nodename();
3201         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3202         $spice_port = PVE::Tools::next_spice_port($pfamily);
3203
3204         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=HIGH,seamless-migration=on";
3205
3206         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3207         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3208         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3209     }
3210
3211     # enable balloon by default, unless explicitly disabled
3212     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3213         $pciaddr = print_pci_addr("balloon0", $bridges);
3214         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3215     }
3216
3217     if ($conf->{watchdog}) {
3218         my $wdopts = parse_watchdog($conf->{watchdog});
3219         $pciaddr = print_pci_addr("watchdog", $bridges);
3220         my $watchdog = $wdopts->{model} || 'i6300esb';
3221         push @$devices, '-device', "$watchdog$pciaddr";
3222         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3223     }
3224
3225     my $vollist = [];
3226     my $scsicontroller = {};
3227     my $ahcicontroller = {};
3228     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3229
3230     # Add iscsi initiator name if available
3231     if (my $initiator = get_initiator_name()) {
3232         push @$devices, '-iscsi', "initiator-name=$initiator";
3233     }
3234
3235     foreach_drive($conf, sub {
3236         my ($ds, $drive) = @_;
3237
3238         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3239             push @$vollist, $drive->{file};
3240         }
3241
3242         $use_virtio = 1 if $ds =~ m/^virtio/;
3243
3244         if (drive_is_cdrom ($drive)) {
3245             if ($bootindex_hash->{d}) {
3246                 $drive->{bootindex} = $bootindex_hash->{d};
3247                 $bootindex_hash->{d} += 1;
3248             }
3249         } else {
3250             if ($bootindex_hash->{c}) {
3251                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3252                 $bootindex_hash->{c} += 1;
3253             }
3254         }
3255
3256         if($drive->{interface} eq 'virtio'){
3257            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3258         }
3259
3260         if ($drive->{interface} eq 'scsi') {
3261
3262             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3263
3264             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3265             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3266
3267             my $iothread = '';
3268             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3269                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3270                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3271             } elsif ($drive->{iothread}) {
3272                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3273             }
3274
3275             my $queues = '';
3276             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3277                 $queues = ",num_queues=$drive->{queues}";
3278             }
3279
3280             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3281             $scsicontroller->{$controller}=1;
3282         }
3283
3284         if ($drive->{interface} eq 'sata') {
3285            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3286            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3287            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3288            $ahcicontroller->{$controller}=1;
3289         }
3290
3291         if ($drive->{interface} eq 'efidisk') {
3292             # this will be added somewhere else
3293             return;
3294         }
3295
3296         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3297         push @$devices, '-drive',$drive_cmd;
3298         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3299     });
3300
3301     for (my $i = 0; $i < $MAX_NETS; $i++) {
3302          next if !$conf->{"net$i"};
3303          my $d = parse_net($conf->{"net$i"});
3304          next if !$d;
3305
3306          $use_virtio = 1 if $d->{model} eq 'virtio';
3307
3308          if ($bootindex_hash->{n}) {
3309             $d->{bootindex} = $bootindex_hash->{n};
3310             $bootindex_hash->{n} += 1;
3311          }
3312
3313          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3314          push @$devices, '-netdev', $netdevfull;
3315
3316          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3317          push @$devices, '-device', $netdevicefull;
3318     }
3319
3320     if (!$q35) {
3321         # add pci bridges
3322         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3323            $bridges->{1} = 1;
3324            $bridges->{2} = 1;
3325         }
3326
3327         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3328
3329         while (my ($k, $v) = each %$bridges) {
3330             $pciaddr = print_pci_addr("pci.$k");
3331             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3332         }
3333     }
3334
3335     # add custom args
3336     if ($conf->{args}) {
3337         my $aa = PVE::Tools::split_args($conf->{args});
3338         push @$cmd, @$aa;
3339     }
3340
3341     push @$cmd, @$devices;
3342     push @$cmd, '-rtc', join(',', @$rtcFlags)
3343         if scalar(@$rtcFlags);
3344     push @$cmd, '-machine', join(',', @$machineFlags)
3345         if scalar(@$machineFlags);
3346     push @$cmd, '-global', join(',', @$globalFlags)
3347         if scalar(@$globalFlags);
3348
3349     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3350 }
3351
3352 sub vnc_socket {
3353     my ($vmid) = @_;
3354     return "${var_run_tmpdir}/$vmid.vnc";
3355 }
3356
3357 sub spice_port {
3358     my ($vmid) = @_;
3359
3360     my $res = vm_mon_cmd($vmid, 'query-spice');
3361
3362     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3363 }
3364
3365 sub qmp_socket {
3366     my ($vmid, $qga) = @_;
3367     my $sockettype = $qga ? 'qga' : 'qmp';
3368     return "${var_run_tmpdir}/$vmid.$sockettype";
3369 }
3370
3371 sub pidfile_name {
3372     my ($vmid) = @_;
3373     return "${var_run_tmpdir}/$vmid.pid";
3374 }
3375
3376 sub vm_devices_list {
3377     my ($vmid) = @_;
3378
3379     my $res = vm_mon_cmd($vmid, 'query-pci');
3380     my $devices = {};
3381     foreach my $pcibus (@$res) {
3382         foreach my $device (@{$pcibus->{devices}}) {
3383             next if !$device->{'qdev_id'};
3384             if ($device->{'pci_bridge'}) {
3385                 $devices->{$device->{'qdev_id'}} = 1;
3386                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3387                     next if !$bridge_device->{'qdev_id'};
3388                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3389                     $devices->{$device->{'qdev_id'}}++;
3390                 }
3391             } else {
3392                 $devices->{$device->{'qdev_id'}} = 1;
3393             }
3394         }
3395     }
3396
3397     my $resblock = vm_mon_cmd($vmid, 'query-block');
3398     foreach my $block (@$resblock) {
3399         if($block->{device} =~ m/^drive-(\S+)/){
3400                 $devices->{$1} = 1;
3401         }
3402     }
3403
3404     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3405     foreach my $mice (@$resmice) {
3406         if ($mice->{name} eq 'QEMU HID Tablet') {
3407             $devices->{tablet} = 1;
3408             last;
3409         }
3410     }
3411
3412     # for usb devices there is no query-usb
3413     # but we can iterate over the entries in
3414     # qom-list path=/machine/peripheral
3415     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3416     foreach my $per (@$resperipheral) {
3417         if ($per->{name} =~ m/^usb\d+$/) {
3418             $devices->{$per->{name}} = 1;
3419         }
3420     }
3421
3422     return $devices;
3423 }
3424
3425 sub vm_deviceplug {
3426     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3427
3428     my $q35 = machine_type_is_q35($conf);
3429
3430     my $devices_list = vm_devices_list($vmid);
3431     return 1 if defined($devices_list->{$deviceid});
3432
3433     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3434
3435     if ($deviceid eq 'tablet') {
3436
3437         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3438
3439     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3440
3441         die "usb hotplug currently not reliable\n";
3442         # since we can't reliably hot unplug all added usb devices
3443         # and usb passthrough disables live migration
3444         # we disable usb hotplugging for now
3445         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3446
3447     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3448
3449         qemu_iothread_add($vmid, $deviceid, $device);
3450
3451         qemu_driveadd($storecfg, $vmid, $device);
3452         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3453
3454         qemu_deviceadd($vmid, $devicefull);
3455         eval { qemu_deviceaddverify($vmid, $deviceid); };
3456         if (my $err = $@) {
3457             eval { qemu_drivedel($vmid, $deviceid); };
3458             warn $@ if $@;
3459             die $err;
3460         }
3461
3462     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3463
3464
3465         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3466         my $pciaddr = print_pci_addr($deviceid);
3467         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3468
3469         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3470
3471         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3472             qemu_iothread_add($vmid, $deviceid, $device);
3473             $devicefull .= ",iothread=iothread-$deviceid";
3474         }
3475
3476         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3477             $devicefull .= ",num_queues=$device->{queues}";
3478         }
3479
3480         qemu_deviceadd($vmid, $devicefull);
3481         qemu_deviceaddverify($vmid, $deviceid);
3482
3483     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3484
3485         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3486         qemu_driveadd($storecfg, $vmid, $device);
3487
3488         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3489         eval { qemu_deviceadd($vmid, $devicefull); };
3490         if (my $err = $@) {
3491             eval { qemu_drivedel($vmid, $deviceid); };
3492             warn $@ if $@;
3493             die $err;
3494         }
3495
3496     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3497
3498         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3499
3500         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3501         my $use_old_bios_files = undef;
3502         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3503
3504         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3505         qemu_deviceadd($vmid, $netdevicefull);
3506         eval { qemu_deviceaddverify($vmid, $deviceid); };
3507         if (my $err = $@) {
3508             eval { qemu_netdevdel($vmid, $deviceid); };
3509             warn $@ if $@;
3510             die $err;
3511         }
3512
3513     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3514
3515         my $bridgeid = $2;
3516         my $pciaddr = print_pci_addr($deviceid);
3517         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3518
3519         qemu_deviceadd($vmid, $devicefull);
3520         qemu_deviceaddverify($vmid, $deviceid);
3521
3522     } else {
3523         die "can't hotplug device '$deviceid'\n";
3524     }
3525
3526     return 1;
3527 }
3528
3529 # fixme: this should raise exceptions on error!
3530 sub vm_deviceunplug {
3531     my ($vmid, $conf, $deviceid) = @_;
3532
3533     my $devices_list = vm_devices_list($vmid);
3534     return 1 if !defined($devices_list->{$deviceid});
3535
3536     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3537
3538     if ($deviceid eq 'tablet') {
3539
3540         qemu_devicedel($vmid, $deviceid);
3541
3542     } elsif ($deviceid =~ m/^usb\d+$/) {
3543
3544         die "usb hotplug currently not reliable\n";
3545         # when unplugging usb devices this way,
3546         # there may be remaining usb controllers/hubs
3547         # so we disable it for now
3548         qemu_devicedel($vmid, $deviceid);
3549         qemu_devicedelverify($vmid, $deviceid);
3550
3551     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3552
3553         qemu_devicedel($vmid, $deviceid);
3554         qemu_devicedelverify($vmid, $deviceid);
3555         qemu_drivedel($vmid, $deviceid);
3556         qemu_iothread_del($conf, $vmid, $deviceid);
3557
3558     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3559
3560         qemu_devicedel($vmid, $deviceid);
3561         qemu_devicedelverify($vmid, $deviceid);
3562         qemu_iothread_del($conf, $vmid, $deviceid);
3563
3564     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3565
3566         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3567         my $device = parse_drive($deviceid, $conf->{$deviceid});
3568         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3569
3570         qemu_devicedel($vmid, $deviceid);
3571         qemu_drivedel($vmid, $deviceid);
3572         qemu_deletescsihw($conf, $vmid, $deviceid);
3573
3574     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3575
3576         qemu_devicedel($vmid, $deviceid);
3577         qemu_devicedelverify($vmid, $deviceid);
3578         qemu_netdevdel($vmid, $deviceid);
3579
3580     } else {
3581         die "can't unplug device '$deviceid'\n";
3582     }
3583
3584     return 1;
3585 }
3586
3587 sub qemu_deviceadd {
3588     my ($vmid, $devicefull) = @_;
3589
3590     $devicefull = "driver=".$devicefull;
3591     my %options =  split(/[=,]/, $devicefull);
3592
3593     vm_mon_cmd($vmid, "device_add" , %options);
3594 }
3595
3596 sub qemu_devicedel {
3597     my ($vmid, $deviceid) = @_;
3598
3599     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3600 }
3601
3602 sub qemu_iothread_add {
3603     my($vmid, $deviceid, $device) = @_;
3604
3605     if ($device->{iothread}) {
3606         my $iothreads = vm_iothreads_list($vmid);
3607         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3608     }
3609 }
3610
3611 sub qemu_iothread_del {
3612     my($conf, $vmid, $deviceid) = @_;
3613
3614     my $device = parse_drive($deviceid, $conf->{$deviceid});
3615     if ($device->{iothread}) {
3616         my $iothreads = vm_iothreads_list($vmid);
3617         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3618     }
3619 }
3620
3621 sub qemu_objectadd {
3622     my($vmid, $objectid, $qomtype) = @_;
3623
3624     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3625
3626     return 1;
3627 }
3628
3629 sub qemu_objectdel {
3630     my($vmid, $objectid) = @_;
3631
3632     vm_mon_cmd($vmid, "object-del", id => $objectid);
3633
3634     return 1;
3635 }
3636
3637 sub qemu_driveadd {
3638     my ($storecfg, $vmid, $device) = @_;
3639
3640     my $drive = print_drive_full($storecfg, $vmid, $device);
3641     $drive =~ s/\\/\\\\/g;
3642     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3643
3644     # If the command succeeds qemu prints: "OK"
3645     return 1 if $ret =~ m/OK/s;
3646
3647     die "adding drive failed: $ret\n";
3648 }
3649
3650 sub qemu_drivedel {
3651     my($vmid, $deviceid) = @_;
3652
3653     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3654     $ret =~ s/^\s+//;
3655
3656     return 1 if $ret eq "";
3657
3658     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3659     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3660
3661     die "deleting drive $deviceid failed : $ret\n";
3662 }
3663
3664 sub qemu_deviceaddverify {
3665     my ($vmid, $deviceid) = @_;
3666
3667     for (my $i = 0; $i <= 5; $i++) {
3668          my $devices_list = vm_devices_list($vmid);
3669          return 1 if defined($devices_list->{$deviceid});
3670          sleep 1;
3671     }
3672
3673     die "error on hotplug device '$deviceid'\n";
3674 }
3675
3676
3677 sub qemu_devicedelverify {
3678     my ($vmid, $deviceid) = @_;
3679
3680     # need to verify that the device is correctly removed as device_del
3681     # is async and empty return is not reliable
3682
3683     for (my $i = 0; $i <= 5; $i++) {
3684          my $devices_list = vm_devices_list($vmid);
3685          return 1 if !defined($devices_list->{$deviceid});
3686          sleep 1;
3687     }
3688
3689     die "error on hot-unplugging device '$deviceid'\n";
3690 }
3691
3692 sub qemu_findorcreatescsihw {
3693     my ($storecfg, $conf, $vmid, $device) = @_;
3694
3695     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3696
3697     my $scsihwid="$controller_prefix$controller";
3698     my $devices_list = vm_devices_list($vmid);
3699
3700     if(!defined($devices_list->{$scsihwid})) {
3701         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3702     }
3703
3704     return 1;
3705 }
3706
3707 sub qemu_deletescsihw {
3708     my ($conf, $vmid, $opt) = @_;
3709
3710     my $device = parse_drive($opt, $conf->{$opt});
3711
3712     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3713         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3714         return 1;
3715     }
3716
3717     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3718
3719     my $devices_list = vm_devices_list($vmid);
3720     foreach my $opt (keys %{$devices_list}) {
3721         if (PVE::QemuServer::is_valid_drivename($opt)) {
3722             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3723             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3724                 return 1;
3725             }
3726         }
3727     }
3728
3729     my $scsihwid="scsihw$controller";
3730
3731     vm_deviceunplug($vmid, $conf, $scsihwid);
3732
3733     return 1;
3734 }
3735
3736 sub qemu_add_pci_bridge {
3737     my ($storecfg, $conf, $vmid, $device) = @_;
3738
3739     my $bridges = {};
3740
3741     my $bridgeid;
3742
3743     print_pci_addr($device, $bridges);
3744
3745     while (my ($k, $v) = each %$bridges) {
3746         $bridgeid = $k;
3747     }
3748     return 1 if !defined($bridgeid) || $bridgeid < 1;
3749
3750     my $bridge = "pci.$bridgeid";
3751     my $devices_list = vm_devices_list($vmid);
3752
3753     if (!defined($devices_list->{$bridge})) {
3754         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3755     }
3756
3757     return 1;
3758 }
3759
3760 sub qemu_set_link_status {
3761     my ($vmid, $device, $up) = @_;
3762
3763     vm_mon_cmd($vmid, "set_link", name => $device,
3764                up => $up ? JSON::true : JSON::false);
3765 }
3766
3767 sub qemu_netdevadd {
3768     my ($vmid, $conf, $device, $deviceid) = @_;
3769
3770     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3771     my %options =  split(/[=,]/, $netdev);
3772
3773     vm_mon_cmd($vmid, "netdev_add",  %options);
3774     return 1;
3775 }
3776
3777 sub qemu_netdevdel {
3778     my ($vmid, $deviceid) = @_;
3779
3780     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3781 }
3782
3783 sub qemu_usb_hotplug {
3784     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3785
3786     return if !$device;
3787
3788     # remove the old one first
3789     vm_deviceunplug($vmid, $conf, $deviceid);
3790
3791     # check if xhci controller is necessary and available
3792     if ($device->{usb3}) {
3793
3794         my $devicelist = vm_devices_list($vmid);
3795
3796         if (!$devicelist->{xhci}) {
3797             my $pciaddr = print_pci_addr("xhci");
3798             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3799         }
3800     }
3801     my $d = parse_usb_device($device->{host});
3802     $d->{usb3} = $device->{usb3};
3803
3804     # add the new one
3805     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3806 }
3807
3808 sub qemu_cpu_hotplug {
3809     my ($vmid, $conf, $vcpus) = @_;
3810
3811     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3812
3813     my $sockets = 1;
3814     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3815     $sockets = $conf->{sockets} if  $conf->{sockets};
3816     my $cores = $conf->{cores} || 1;
3817     my $maxcpus = $sockets * $cores;
3818
3819     $vcpus = $maxcpus if !$vcpus;
3820
3821     die "you can't add more vcpus than maxcpus\n"
3822         if $vcpus > $maxcpus;
3823
3824     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3825
3826     if ($vcpus < $currentvcpus) {
3827
3828         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3829
3830             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3831                 qemu_devicedel($vmid, "cpu$i");
3832                 my $retry = 0;
3833                 my $currentrunningvcpus = undef;
3834                 while (1) {
3835                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3836                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3837                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3838                     $retry++;
3839                     sleep 1;
3840                 }
3841                 #update conf after each succesfull cpu unplug
3842                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3843                 PVE::QemuConfig->write_config($vmid, $conf);
3844             }
3845         } else {
3846             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3847         }
3848
3849         return;
3850     }
3851
3852     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3853     die "vcpus in running vm does not match its configuration\n"
3854         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3855
3856     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3857
3858         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3859             my $cpustr = print_cpu_device($conf, $i);
3860             qemu_deviceadd($vmid, $cpustr);
3861
3862             my $retry = 0;
3863             my $currentrunningvcpus = undef;
3864             while (1) {
3865                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3866                 last if scalar(@{$currentrunningvcpus}) == $i;
3867                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3868                 sleep 1;
3869                 $retry++;
3870             }
3871             #update conf after each succesfull cpu hotplug
3872             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3873             PVE::QemuConfig->write_config($vmid, $conf);
3874         }
3875     } else {
3876
3877         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3878             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3879         }
3880     }
3881 }
3882
3883 sub qemu_block_set_io_throttle {
3884     my ($vmid, $deviceid,
3885         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3886         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3887         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3888         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3889
3890     return if !check_running($vmid) ;
3891
3892     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3893         bps => int($bps),
3894         bps_rd => int($bps_rd),
3895         bps_wr => int($bps_wr),
3896         iops => int($iops),
3897         iops_rd => int($iops_rd),
3898         iops_wr => int($iops_wr),
3899         bps_max => int($bps_max),
3900         bps_rd_max => int($bps_rd_max),
3901         bps_wr_max => int($bps_wr_max),
3902         iops_max => int($iops_max),
3903         iops_rd_max => int($iops_rd_max),
3904         iops_wr_max => int($iops_wr_max),
3905         bps_max_length => int($bps_max_length),
3906         bps_rd_max_length => int($bps_rd_max_length),
3907         bps_wr_max_length => int($bps_wr_max_length),
3908         iops_max_length => int($iops_max_length),
3909         iops_rd_max_length => int($iops_rd_max_length),
3910         iops_wr_max_length => int($iops_wr_max_length),
3911     );
3912
3913 }
3914
3915 # old code, only used to shutdown old VM after update
3916 sub __read_avail {
3917     my ($fh, $timeout) = @_;
3918
3919     my $sel = new IO::Select;
3920     $sel->add($fh);
3921
3922     my $res = '';
3923     my $buf;
3924
3925     my @ready;
3926     while (scalar (@ready = $sel->can_read($timeout))) {
3927         my $count;
3928         if ($count = $fh->sysread($buf, 8192)) {
3929             if ($buf =~ /^(.*)\(qemu\) $/s) {
3930                 $res .= $1;
3931                 last;
3932             } else {
3933                 $res .= $buf;
3934             }
3935         } else {
3936             if (!defined($count)) {
3937                 die "$!\n";
3938             }
3939             last;
3940         }
3941     }
3942
3943     die "monitor read timeout\n" if !scalar(@ready);
3944
3945     return $res;
3946 }
3947
3948 # old code, only used to shutdown old VM after update
3949 sub vm_monitor_command {
3950     my ($vmid, $cmdstr, $nocheck) = @_;
3951
3952     my $res;
3953
3954     eval {
3955         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3956
3957         my $sname = "${var_run_tmpdir}/$vmid.mon";
3958
3959         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3960             die "unable to connect to VM $vmid socket - $!\n";
3961
3962         my $timeout = 3;
3963
3964         # hack: migrate sometime blocks the monitor (when migrate_downtime
3965         # is set)
3966         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3967             $timeout = 60*60; # 1 hour
3968         }
3969
3970         # read banner;
3971         my $data = __read_avail($sock, $timeout);
3972
3973         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3974             die "got unexpected qemu monitor banner\n";
3975         }
3976
3977         my $sel = new IO::Select;
3978         $sel->add($sock);
3979
3980         if (!scalar(my @ready = $sel->can_write($timeout))) {
3981             die "monitor write error - timeout";
3982         }
3983
3984         my $fullcmd = "$cmdstr\r";
3985
3986         # syslog('info', "VM $vmid monitor command: $cmdstr");
3987
3988         my $b;
3989         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3990             die "monitor write error - $!";
3991         }
3992
3993         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3994
3995         $timeout = 20;
3996
3997         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3998             $timeout = 60*60; # 1 hour
3999         } elsif ($cmdstr =~ m/^(eject|change)/) {
4000             $timeout = 60; # note: cdrom mount command is slow
4001         }
4002         if ($res = __read_avail($sock, $timeout)) {
4003
4004             my @lines = split("\r?\n", $res);
4005
4006             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4007
4008             $res = join("\n", @lines);
4009             $res .= "\n";
4010         }
4011     };
4012
4013     my $err = $@;
4014
4015     if ($err) {
4016         syslog("err", "VM $vmid monitor command failed - $err");
4017         die $err;
4018     }
4019
4020     return $res;
4021 }
4022
4023 sub qemu_block_resize {
4024     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4025
4026     my $running = check_running($vmid);
4027
4028     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4029
4030     return if !$running;
4031
4032     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4033
4034 }
4035
4036 sub qemu_volume_snapshot {
4037     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4038
4039     my $running = check_running($vmid);
4040
4041     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4042         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4043     } else {
4044         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4045     }
4046 }
4047
4048 sub qemu_volume_snapshot_delete {
4049     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4050
4051     my $running = check_running($vmid);
4052
4053     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4054         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4055     } else {
4056         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4057     }
4058 }
4059
4060 sub set_migration_caps {
4061     my ($vmid) = @_;
4062
4063     my $cap_ref = [];
4064
4065     my $enabled_cap = {
4066         "auto-converge" => 1,
4067         "xbzrle" => 1,
4068         "x-rdma-pin-all" => 0,
4069         "zero-blocks" => 0,
4070         "compress" => 0
4071     };
4072
4073     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4074
4075     for my $supported_capability (@$supported_capabilities) {
4076         push @$cap_ref, {
4077             capability => $supported_capability->{capability},
4078             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4079         };
4080     }
4081
4082     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4083 }
4084
4085 my $fast_plug_option = {
4086     'lock' => 1,
4087     'name' => 1,
4088     'onboot' => 1,
4089     'shares' => 1,
4090     'startup' => 1,
4091     'description' => 1,
4092     'protection' => 1,
4093 };
4094
4095 # hotplug changes in [PENDING]
4096 # $selection hash can be used to only apply specified options, for
4097 # example: { cores => 1 } (only apply changed 'cores')
4098 # $errors ref is used to return error messages
4099 sub vmconfig_hotplug_pending {
4100     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4101
4102     my $defaults = load_defaults();
4103
4104     # commit values which do not have any impact on running VM first
4105     # Note: those option cannot raise errors, we we do not care about
4106     # $selection and always apply them.
4107
4108     my $add_error = sub {
4109         my ($opt, $msg) = @_;
4110         $errors->{$opt} = "hotplug problem - $msg";
4111     };
4112
4113     my $changes = 0;
4114     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4115         if ($fast_plug_option->{$opt}) {
4116             $conf->{$opt} = $conf->{pending}->{$opt};
4117             delete $conf->{pending}->{$opt};
4118             $changes = 1;
4119         }
4120     }
4121
4122     if ($changes) {
4123         PVE::QemuConfig->write_config($vmid, $conf);
4124         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4125     }
4126
4127     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4128
4129     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4130     while (my ($opt, $force) = each %$pending_delete_hash) {
4131         next if $selection && !$selection->{$opt};
4132         eval {
4133             if ($opt eq 'hotplug') {
4134                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4135             } elsif ($opt eq 'tablet') {
4136                 die "skip\n" if !$hotplug_features->{usb};
4137                 if ($defaults->{tablet}) {
4138                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4139                 } else {
4140                     vm_deviceunplug($vmid, $conf, $opt);
4141                 }
4142             } elsif ($opt =~ m/^usb\d+/) {
4143                 die "skip\n";
4144                 # since we cannot reliably hot unplug usb devices
4145                 # we are disabling it
4146                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4147                 vm_deviceunplug($vmid, $conf, $opt);
4148             } elsif ($opt eq 'vcpus') {
4149                 die "skip\n" if !$hotplug_features->{cpu};
4150                 qemu_cpu_hotplug($vmid, $conf, undef);
4151             } elsif ($opt eq 'balloon') {
4152                 # enable balloon device is not hotpluggable
4153                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4154             } elsif ($fast_plug_option->{$opt}) {
4155                 # do nothing
4156             } elsif ($opt =~ m/^net(\d+)$/) {
4157                 die "skip\n" if !$hotplug_features->{network};
4158                 vm_deviceunplug($vmid, $conf, $opt);
4159             } elsif (is_valid_drivename($opt)) {
4160                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4161                 vm_deviceunplug($vmid, $conf, $opt);
4162                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4163             } elsif ($opt =~ m/^memory$/) {
4164                 die "skip\n" if !$hotplug_features->{memory};
4165                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4166             } elsif ($opt eq 'cpuunits') {
4167                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4168             } elsif ($opt eq 'cpulimit') {
4169                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4170             } else {
4171                 die "skip\n";
4172             }
4173         };
4174         if (my $err = $@) {
4175             &$add_error($opt, $err) if $err ne "skip\n";
4176         } else {
4177             # save new config if hotplug was successful
4178             delete $conf->{$opt};
4179             vmconfig_undelete_pending_option($conf, $opt);
4180             PVE::QemuConfig->write_config($vmid, $conf);
4181             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4182         }
4183     }
4184
4185     foreach my $opt (keys %{$conf->{pending}}) {
4186         next if $selection && !$selection->{$opt};
4187         my $value = $conf->{pending}->{$opt};
4188         eval {
4189             if ($opt eq 'hotplug') {
4190                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4191             } elsif ($opt eq 'tablet') {
4192                 die "skip\n" if !$hotplug_features->{usb};
4193                 if ($value == 1) {
4194                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4195                 } elsif ($value == 0) {
4196                     vm_deviceunplug($vmid, $conf, $opt);
4197                 }
4198             } elsif ($opt =~ m/^usb\d+$/) {
4199                 die "skip\n";
4200                 # since we cannot reliably hot unplug usb devices
4201                 # we are disabling it
4202                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4203                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4204                 die "skip\n" if !$d;
4205                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4206             } elsif ($opt eq 'vcpus') {
4207                 die "skip\n" if !$hotplug_features->{cpu};
4208                 qemu_cpu_hotplug($vmid, $conf, $value);
4209             } elsif ($opt eq 'balloon') {
4210                 # enable/disable balloning device is not hotpluggable
4211                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4212                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4213                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4214
4215                 # allow manual ballooning if shares is set to zero
4216                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4217                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4218                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4219                 }
4220             } elsif ($opt =~ m/^net(\d+)$/) {
4221                 # some changes can be done without hotplug
4222                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4223                                     $vmid, $opt, $value);
4224             } elsif (is_valid_drivename($opt)) {
4225                 # some changes can be done without hotplug
4226                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4227                                      $vmid, $opt, $value, 1);
4228             } elsif ($opt =~ m/^memory$/) { #dimms
4229                 die "skip\n" if !$hotplug_features->{memory};
4230                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4231             } elsif ($opt eq 'cpuunits') {
4232                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4233             } elsif ($opt eq 'cpulimit') {
4234                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4235                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4236             } else {
4237                 die "skip\n";  # skip non-hot-pluggable options
4238             }
4239         };
4240         if (my $err = $@) {
4241             &$add_error($opt, $err) if $err ne "skip\n";
4242         } else {
4243             # save new config if hotplug was successful
4244             $conf->{$opt} = $value;
4245             delete $conf->{pending}->{$opt};
4246             PVE::QemuConfig->write_config($vmid, $conf);
4247             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4248         }
4249     }
4250 }
4251
4252 sub try_deallocate_drive {
4253     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4254
4255     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4256         my $volid = $drive->{file};
4257         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4258             my $sid = PVE::Storage::parse_volume_id($volid);
4259             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4260
4261             # check if the disk is really unused
4262             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4263                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4264             PVE::Storage::vdisk_free($storecfg, $volid);
4265             return 1;
4266         } else {
4267             # If vm is not owner of this disk remove from config
4268             return 1;
4269         }
4270     }
4271
4272     return undef;
4273 }
4274
4275 sub vmconfig_delete_or_detach_drive {
4276     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4277
4278     my $drive = parse_drive($opt, $conf->{$opt});
4279
4280     my $rpcenv = PVE::RPCEnvironment::get();
4281     my $authuser = $rpcenv->get_user();
4282
4283     if ($force) {
4284         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4285         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4286     } else {
4287         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4288     }
4289 }
4290
4291 sub vmconfig_apply_pending {
4292     my ($vmid, $conf, $storecfg) = @_;
4293
4294     # cold plug
4295
4296     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4297     while (my ($opt, $force) = each %$pending_delete_hash) {
4298         die "internal error" if $opt =~ m/^unused/;
4299         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4300         if (!defined($conf->{$opt})) {
4301             vmconfig_undelete_pending_option($conf, $opt);
4302             PVE::QemuConfig->write_config($vmid, $conf);
4303         } elsif (is_valid_drivename($opt)) {
4304             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4305             vmconfig_undelete_pending_option($conf, $opt);
4306             delete $conf->{$opt};
4307             PVE::QemuConfig->write_config($vmid, $conf);
4308         } else {
4309             vmconfig_undelete_pending_option($conf, $opt);
4310             delete $conf->{$opt};
4311             PVE::QemuConfig->write_config($vmid, $conf);
4312         }
4313     }
4314
4315     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4316
4317     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4318         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4319
4320         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4321             # skip if nothing changed
4322         } elsif (is_valid_drivename($opt)) {
4323             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4324                 if defined($conf->{$opt});
4325             $conf->{$opt} = $conf->{pending}->{$opt};
4326         } else {
4327             $conf->{$opt} = $conf->{pending}->{$opt};
4328         }
4329
4330         delete $conf->{pending}->{$opt};
4331         PVE::QemuConfig->write_config($vmid, $conf);
4332     }
4333 }
4334
4335 my $safe_num_ne = sub {
4336     my ($a, $b) = @_;
4337
4338     return 0 if !defined($a) && !defined($b);
4339     return 1 if !defined($a);
4340     return 1 if !defined($b);
4341
4342     return $a != $b;
4343 };
4344
4345 my $safe_string_ne = sub {
4346     my ($a, $b) = @_;
4347
4348     return 0 if !defined($a) && !defined($b);
4349     return 1 if !defined($a);
4350     return 1 if !defined($b);
4351
4352     return $a ne $b;
4353 };
4354
4355 sub vmconfig_update_net {
4356     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4357
4358     my $newnet = parse_net($value);
4359
4360     if ($conf->{$opt}) {
4361         my $oldnet = parse_net($conf->{$opt});
4362
4363         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4364             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4365             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4366             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4367
4368             # for non online change, we try to hot-unplug
4369             die "skip\n" if !$hotplug;
4370             vm_deviceunplug($vmid, $conf, $opt);
4371         } else {
4372
4373             die "internal error" if $opt !~ m/net(\d+)/;
4374             my $iface = "tap${vmid}i$1";
4375
4376             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4377                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4378                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4379                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4380                 PVE::Network::tap_unplug($iface);
4381                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4382             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4383                 # Rate can be applied on its own but any change above needs to
4384                 # include the rate in tap_plug since OVS resets everything.
4385                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4386             }
4387
4388             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4389                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4390             }
4391
4392             return 1;
4393         }
4394     }
4395
4396     if ($hotplug) {
4397         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4398     } else {
4399         die "skip\n";
4400     }
4401 }
4402
4403 sub vmconfig_update_disk {
4404     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4405
4406     # fixme: do we need force?
4407
4408     my $drive = parse_drive($opt, $value);
4409
4410     if ($conf->{$opt}) {
4411
4412         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4413
4414             my $media = $drive->{media} || 'disk';
4415             my $oldmedia = $old_drive->{media} || 'disk';
4416             die "unable to change media type\n" if $media ne $oldmedia;
4417
4418             if (!drive_is_cdrom($old_drive)) {
4419
4420                 if ($drive->{file} ne $old_drive->{file}) {
4421
4422                     die "skip\n" if !$hotplug;
4423
4424                     # unplug and register as unused
4425                     vm_deviceunplug($vmid, $conf, $opt);
4426                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4427
4428                 } else {
4429                     # update existing disk
4430
4431                     # skip non hotpluggable value
4432                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4433                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4434                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4435                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4436                         die "skip\n";
4437                     }
4438
4439                     # apply throttle
4440                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4441                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4442                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4443                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4444                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4445                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4446                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4447                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4448                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4449                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4450                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4451                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4452                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4453                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4454                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4455                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4456                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4457                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4458
4459                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4460                                                    ($drive->{mbps} || 0)*1024*1024,
4461                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4462                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4463                                                    $drive->{iops} || 0,
4464                                                    $drive->{iops_rd} || 0,
4465                                                    $drive->{iops_wr} || 0,
4466                                                    ($drive->{mbps_max} || 0)*1024*1024,
4467                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4468                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4469                                                    $drive->{iops_max} || 0,
4470                                                    $drive->{iops_rd_max} || 0,
4471                                                    $drive->{iops_wr_max} || 0,
4472                                                    $drive->{bps_max_length} || 1,
4473                                                    $drive->{bps_rd_max_length} || 1,
4474                                                    $drive->{bps_wr_max_length} || 1,
4475                                                    $drive->{iops_max_length} || 1,
4476                                                    $drive->{iops_rd_max_length} || 1,
4477                                                    $drive->{iops_wr_max_length} || 1);
4478
4479                     }
4480
4481                     return 1;
4482                 }
4483
4484             } else { # cdrom
4485
4486                 if ($drive->{file} eq 'none') {
4487                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4488                 } else {
4489                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4490                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4491                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4492                 }
4493
4494                 return 1;
4495             }
4496         }
4497     }
4498
4499     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4500     # hotplug new disks
4501     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4502     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4503 }
4504
4505 sub vm_start {
4506     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4507         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4508
4509     PVE::QemuConfig->lock_config($vmid, sub {
4510         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4511
4512         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4513
4514         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4515
4516         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4517
4518         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4519             vmconfig_apply_pending($vmid, $conf, $storecfg);
4520             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4521         }
4522
4523         my $defaults = load_defaults();
4524
4525         # set environment variable useful inside network script
4526         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4527
4528         my $local_volumes = {};
4529
4530         if ($targetstorage) {
4531             foreach_drive($conf, sub {
4532                 my ($ds, $drive) = @_;
4533
4534                 return if drive_is_cdrom($drive);
4535
4536                 my $volid = $drive->{file};
4537
4538                 return if !$volid;
4539
4540                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4541
4542                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4543                 return if $scfg->{shared};
4544                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4545             });
4546
4547             my $format = undef;
4548
4549             foreach my $opt (sort keys %$local_volumes) {
4550
4551                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4552                 my $drive = parse_drive($opt, $conf->{$opt});
4553
4554                 #if remote storage is specified, use default format
4555                 if ($targetstorage && $targetstorage ne "1") {
4556                     $storeid = $targetstorage;
4557                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4558                     $format = $defFormat;
4559                 } else {
4560                     #else we use same format than original
4561                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4562                     $format = qemu_img_format($scfg, $volid);
4563                 }
4564
4565                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4566                 my $newdrive = $drive;
4567                 $newdrive->{format} = $format;
4568                 $newdrive->{file} = $newvolid;
4569                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4570                 $local_volumes->{$opt} = $drivestr;
4571                 #pass drive to conf for command line
4572                 $conf->{$opt} = $drivestr;
4573             }
4574         }
4575
4576         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4577
4578         my $migrate_port = 0;
4579         my $migrate_uri;
4580         if ($statefile) {
4581             if ($statefile eq 'tcp') {
4582                 my $localip = "localhost";
4583                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4584                 my $nodename = PVE::INotify::nodename();
4585
4586                 if (!defined($migration_type)) {
4587                     if (defined($datacenterconf->{migration}->{type})) {
4588                         $migration_type = $datacenterconf->{migration}->{type};
4589                     } else {
4590                         $migration_type = 'secure';
4591                     }
4592                 }
4593
4594                 if ($migration_type eq 'insecure') {
4595                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4596                     if ($migrate_network_addr) {
4597                         $localip = $migrate_network_addr;
4598                     } else {
4599                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4600                     }
4601
4602                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4603                 }
4604
4605                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4606                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4607                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4608                 push @$cmd, '-incoming', $migrate_uri;
4609                 push @$cmd, '-S';
4610
4611             } elsif ($statefile eq 'unix') {
4612                 # should be default for secure migrations as a ssh TCP forward
4613                 # tunnel is not deterministic reliable ready and fails regurarly
4614                 # to set up in time, so use UNIX socket forwards
4615                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4616                 unlink $socket_addr;
4617
4618                 $migrate_uri = "unix:$socket_addr";
4619
4620                 push @$cmd, '-incoming', $migrate_uri;
4621                 push @$cmd, '-S';
4622
4623             } else {
4624                 push @$cmd, '-loadstate', $statefile;
4625             }
4626         } elsif ($paused) {
4627             push @$cmd, '-S';
4628         }
4629
4630         # host pci devices
4631         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4632           my $d = parse_hostpci($conf->{"hostpci$i"});
4633           next if !$d;
4634           my $pcidevices = $d->{pciid};
4635           foreach my $pcidevice (@$pcidevices) {
4636                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4637
4638                 my $info = pci_device_info("0000:$pciid");
4639                 die "IOMMU not present\n" if !check_iommu_support();
4640                 die "no pci device info for device '$pciid'\n" if !$info;
4641                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4642                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4643           }
4644         }
4645
4646         PVE::Storage::activate_volumes($storecfg, $vollist);
4647
4648         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4649             my $cmd = [];
4650             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4651             eval  { run_command($cmd); };
4652         }
4653
4654         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4655                                                   : $defaults->{cpuunits};
4656
4657         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4658
4659         my %properties = (
4660             Slice => 'qemu.slice',
4661             KillMode => 'none',
4662             CPUShares => $cpuunits
4663         );
4664
4665         if (my $cpulimit = $conf->{cpulimit}) {
4666             $properties{CPUQuota} = int($cpulimit * 100);
4667         }
4668         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4669
4670         if ($conf->{hugepages}) {
4671
4672             my $code = sub {
4673                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4674                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4675
4676                 PVE::QemuServer::Memory::hugepages_mount();
4677                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4678
4679                 eval  {
4680                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4681                     run_command($cmd, %run_params);
4682                 };
4683
4684                 if (my $err = $@) {
4685                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4686                     die $err;
4687                 }
4688
4689                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4690             };
4691             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4692
4693         } else {
4694             eval  {
4695                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4696                 run_command($cmd, %run_params);
4697             };
4698         }
4699
4700         if (my $err = $@) {
4701             # deactivate volumes if start fails
4702             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4703             die "start failed: $err";
4704         }
4705
4706         print "migration listens on $migrate_uri\n" if $migrate_uri;
4707
4708         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4709             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4710             warn $@ if $@;
4711         }
4712
4713         #start nbd server for storage migration
4714         if ($targetstorage) {
4715             my $nodename = PVE::INotify::nodename();
4716             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4717             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4718             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4719             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4720
4721             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4722
4723             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4724
4725             foreach my $opt (sort keys %$local_volumes) {
4726                 my $volid = $local_volumes->{$opt};
4727                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4728                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4729                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4730             }
4731         }
4732
4733         if ($migratedfrom) {
4734             eval {
4735                 set_migration_caps($vmid);
4736             };
4737             warn $@ if $@;
4738
4739             if ($spice_port) {
4740                 print "spice listens on port $spice_port\n";
4741                 if ($spice_ticket) {
4742                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4743                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4744                 }
4745             }
4746
4747         } else {
4748             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4749                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4750                     if $conf->{balloon};
4751             }
4752
4753             foreach my $opt (keys %$conf) {
4754                 next if $opt !~  m/^net\d+$/;
4755                 my $nicconf = parse_net($conf->{$opt});
4756                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4757             }
4758         }
4759
4760         vm_mon_cmd_nocheck($vmid, 'qom-set',
4761                     path => "machine/peripheral/balloon0",
4762                     property => "guest-stats-polling-interval",
4763                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4764
4765     });
4766 }
4767
4768 sub vm_mon_cmd {
4769     my ($vmid, $execute, %params) = @_;
4770
4771     my $cmd = { execute => $execute, arguments => \%params };
4772     vm_qmp_command($vmid, $cmd);
4773 }
4774
4775 sub vm_mon_cmd_nocheck {
4776     my ($vmid, $execute, %params) = @_;
4777
4778     my $cmd = { execute => $execute, arguments => \%params };
4779     vm_qmp_command($vmid, $cmd, 1);
4780 }
4781
4782 sub vm_qmp_command {
4783     my ($vmid, $cmd, $nocheck) = @_;
4784
4785     my $res;
4786
4787     my $timeout;
4788     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4789         $timeout = $cmd->{arguments}->{timeout};
4790         delete $cmd->{arguments}->{timeout};
4791     }
4792
4793     eval {
4794         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4795         my $sname = qmp_socket($vmid);
4796         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4797             my $qmpclient = PVE::QMPClient->new();
4798
4799             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4800         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4801             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4802                 if scalar(%{$cmd->{arguments}});
4803             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4804         } else {
4805             die "unable to open monitor socket\n";
4806         }
4807     };
4808     if (my $err = $@) {
4809         syslog("err", "VM $vmid qmp command failed - $err");
4810         die $err;
4811     }
4812
4813     return $res;
4814 }
4815
4816 sub vm_human_monitor_command {
4817     my ($vmid, $cmdline) = @_;
4818
4819     my $res;
4820
4821     my $cmd = {
4822         execute => 'human-monitor-command',
4823         arguments => { 'command-line' => $cmdline},
4824     };
4825
4826     return vm_qmp_command($vmid, $cmd);
4827 }
4828
4829 sub vm_commandline {
4830     my ($storecfg, $vmid) = @_;
4831
4832     my $conf = PVE::QemuConfig->load_config($vmid);
4833
4834     my $defaults = load_defaults();
4835
4836     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4837
4838     return PVE::Tools::cmd2string($cmd);
4839 }
4840
4841 sub vm_reset {
4842     my ($vmid, $skiplock) = @_;
4843
4844     PVE::QemuConfig->lock_config($vmid, sub {
4845
4846         my $conf = PVE::QemuConfig->load_config($vmid);
4847
4848         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4849
4850         vm_mon_cmd($vmid, "system_reset");
4851     });
4852 }
4853
4854 sub get_vm_volumes {
4855     my ($conf) = @_;
4856
4857     my $vollist = [];
4858     foreach_volid($conf, sub {
4859         my ($volid, $is_cdrom) = @_;
4860
4861         return if $volid =~ m|^/|;
4862
4863         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4864         return if !$sid;
4865
4866         push @$vollist, $volid;
4867     });
4868
4869     return $vollist;
4870 }
4871
4872 sub vm_stop_cleanup {
4873     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4874
4875     eval {
4876
4877         if (!$keepActive) {
4878             my $vollist = get_vm_volumes($conf);
4879             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4880         }
4881
4882         foreach my $ext (qw(mon qmp pid vnc qga)) {
4883             unlink "/var/run/qemu-server/${vmid}.$ext";
4884         }
4885
4886         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4887     };
4888     warn $@ if $@; # avoid errors - just warn
4889 }
4890
4891 # Note: use $nockeck to skip tests if VM configuration file exists.
4892 # We need that when migration VMs to other nodes (files already moved)
4893 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4894 sub vm_stop {
4895     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4896
4897     $force = 1 if !defined($force) && !$shutdown;
4898
4899     if ($migratedfrom){
4900         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4901         kill 15, $pid if $pid;
4902         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4903         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4904         return;
4905     }
4906
4907     PVE::QemuConfig->lock_config($vmid, sub {
4908
4909         my $pid = check_running($vmid, $nocheck);
4910         return if !$pid;
4911
4912         my $conf;
4913         if (!$nocheck) {
4914             $conf = PVE::QemuConfig->load_config($vmid);
4915             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4916             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4917                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4918                 $timeout = $opts->{down} if $opts->{down};
4919             }
4920         }
4921
4922         $timeout = 60 if !defined($timeout);
4923
4924         eval {
4925             if ($shutdown) {
4926                 if (defined($conf) && $conf->{agent}) {
4927                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4928                 } else {
4929                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4930                 }
4931             } else {
4932                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4933             }
4934         };
4935         my $err = $@;
4936
4937         if (!$err) {
4938             my $count = 0;
4939             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4940                 $count++;
4941                 sleep 1;
4942             }
4943
4944             if ($count >= $timeout) {
4945                 if ($force) {
4946                     warn "VM still running - terminating now with SIGTERM\n";
4947                     kill 15, $pid;
4948                 } else {
4949                     die "VM quit/powerdown failed - got timeout\n";
4950                 }
4951             } else {
4952                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4953                 return;
4954             }
4955         } else {
4956             if ($force) {
4957                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4958                 kill 15, $pid;
4959             } else {
4960                 die "VM quit/powerdown failed\n";
4961             }
4962         }
4963
4964         # wait again
4965         $timeout = 10;
4966
4967         my $count = 0;
4968         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4969             $count++;
4970             sleep 1;
4971         }
4972
4973         if ($count >= $timeout) {
4974             warn "VM still running - terminating now with SIGKILL\n";
4975             kill 9, $pid;
4976             sleep 1;
4977         }
4978
4979         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4980    });
4981 }
4982
4983 sub vm_suspend {
4984     my ($vmid, $skiplock) = @_;
4985
4986     PVE::QemuConfig->lock_config($vmid, sub {
4987
4988         my $conf = PVE::QemuConfig->load_config($vmid);
4989
4990         PVE::QemuConfig->check_lock($conf)
4991             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4992
4993         vm_mon_cmd($vmid, "stop");
4994     });
4995 }
4996
4997 sub vm_resume {
4998     my ($vmid, $skiplock, $nocheck) = @_;
4999
5000     PVE::QemuConfig->lock_config($vmid, sub {
5001
5002         if (!$nocheck) {
5003
5004             my $conf = PVE::QemuConfig->load_config($vmid);
5005
5006             PVE::QemuConfig->check_lock($conf)
5007                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5008
5009             vm_mon_cmd($vmid, "cont");
5010
5011         } else {
5012             vm_mon_cmd_nocheck($vmid, "cont");
5013         }
5014     });
5015 }
5016
5017 sub vm_sendkey {
5018     my ($vmid, $skiplock, $key) = @_;
5019
5020     PVE::QemuConfig->lock_config($vmid, sub {
5021
5022         my $conf = PVE::QemuConfig->load_config($vmid);
5023
5024         # there is no qmp command, so we use the human monitor command
5025         vm_human_monitor_command($vmid, "sendkey $key");
5026     });
5027 }
5028
5029 sub vm_destroy {
5030     my ($storecfg, $vmid, $skiplock) = @_;
5031
5032     PVE::QemuConfig->lock_config($vmid, sub {
5033
5034         my $conf = PVE::QemuConfig->load_config($vmid);
5035
5036         if (!check_running($vmid)) {
5037             destroy_vm($storecfg, $vmid, undef, $skiplock);
5038         } else {
5039             die "VM $vmid is running - destroy failed\n";
5040         }
5041     });
5042 }
5043
5044 # pci helpers
5045
5046 sub file_write {
5047     my ($filename, $buf) = @_;
5048
5049     my $fh = IO::File->new($filename, "w");
5050     return undef if !$fh;
5051
5052     my $res = print $fh $buf;
5053
5054     $fh->close();
5055
5056     return $res;
5057 }
5058
5059 sub pci_device_info {
5060     my ($name) = @_;
5061
5062     my $res;
5063
5064     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5065     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5066
5067     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5068     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5069
5070     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5071     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5072
5073     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5074     return undef if !defined($product) || $product !~ s/^0x//;
5075
5076     $res = {
5077         name => $name,
5078         vendor => $vendor,
5079         product => $product,
5080         domain => $domain,
5081         bus => $bus,
5082         slot => $slot,
5083         func => $func,
5084         irq => $irq,
5085         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5086     };
5087
5088     return $res;
5089 }
5090
5091 sub pci_dev_reset {
5092     my ($dev) = @_;
5093
5094     my $name = $dev->{name};
5095
5096     my $fn = "$pcisysfs/devices/$name/reset";
5097
5098     return file_write($fn, "1");
5099 }
5100
5101 sub pci_dev_bind_to_vfio {
5102     my ($dev) = @_;
5103
5104     my $name = $dev->{name};
5105
5106     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5107
5108     if (!-d $vfio_basedir) {
5109         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5110     }
5111     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5112
5113     my $testdir = "$vfio_basedir/$name";
5114     return 1 if -d $testdir;
5115
5116     my $data = "$dev->{vendor} $dev->{product}";
5117     return undef if !file_write("$vfio_basedir/new_id", $data);
5118
5119     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5120     if (!file_write($fn, $name)) {
5121         return undef if -f $fn;
5122     }
5123
5124     $fn = "$vfio_basedir/bind";
5125     if (! -d $testdir) {
5126         return undef if !file_write($fn, $name);
5127     }
5128
5129     return -d $testdir;
5130 }
5131
5132 sub pci_dev_group_bind_to_vfio {
5133     my ($pciid) = @_;
5134
5135     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5136
5137     if (!-d $vfio_basedir) {
5138         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5139     }
5140     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5141
5142     # get IOMMU group devices
5143     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5144       my @devs = grep /^0000:/, readdir($D);
5145     closedir($D);
5146
5147     foreach my $pciid (@devs) {
5148         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5149
5150         # pci bridges, switches or root ports are not supported
5151         # they have a pci_bus subdirectory so skip them
5152         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5153
5154         my $info = pci_device_info($1);
5155         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5156     }
5157
5158     return 1;
5159 }
5160
5161 # vzdump restore implementaion
5162
5163 sub tar_archive_read_firstfile {
5164     my $archive = shift;
5165
5166     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5167
5168     # try to detect archive type first
5169     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5170         die "unable to open file '$archive'\n";
5171     my $firstfile = <$fh>;
5172     kill 15, $pid;
5173     close $fh;
5174
5175     die "ERROR: archive contaions no data\n" if !$firstfile;
5176     chomp $firstfile;
5177
5178     return $firstfile;
5179 }
5180
5181 sub tar_restore_cleanup {
5182     my ($storecfg, $statfile) = @_;
5183
5184     print STDERR "starting cleanup\n";
5185
5186     if (my $fd = IO::File->new($statfile, "r")) {
5187         while (defined(my $line = <$fd>)) {
5188             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5189                 my $volid = $2;
5190                 eval {
5191                     if ($volid =~ m|^/|) {
5192                         unlink $volid || die 'unlink failed\n';
5193                     } else {
5194                         PVE::Storage::vdisk_free($storecfg, $volid);
5195                     }
5196                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5197                 };
5198                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5199             } else {
5200                 print STDERR "unable to parse line in statfile - $line";
5201             }
5202         }
5203         $fd->close();
5204     }
5205 }
5206
5207 sub restore_archive {
5208     my ($archive, $vmid, $user, $opts) = @_;
5209
5210     my $format = $opts->{format};
5211     my $comp;
5212
5213     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5214         $format = 'tar' if !$format;
5215         $comp = 'gzip';
5216     } elsif ($archive =~ m/\.tar$/) {
5217         $format = 'tar' if !$format;
5218     } elsif ($archive =~ m/.tar.lzo$/) {
5219         $format = 'tar' if !$format;
5220         $comp = 'lzop';
5221     } elsif ($archive =~ m/\.vma$/) {
5222         $format = 'vma' if !$format;
5223     } elsif ($archive =~ m/\.vma\.gz$/) {
5224         $format = 'vma' if !$format;
5225         $comp = 'gzip';
5226     } elsif ($archive =~ m/\.vma\.lzo$/) {
5227         $format = 'vma' if !$format;
5228         $comp = 'lzop';
5229     } else {
5230         $format = 'vma' if !$format; # default
5231     }
5232
5233     # try to detect archive format
5234     if ($format eq 'tar') {
5235         return restore_tar_archive($archive, $vmid, $user, $opts);
5236     } else {
5237         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5238     }
5239 }
5240
5241 sub restore_update_config_line {
5242     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5243
5244     return if $line =~ m/^\#qmdump\#/;
5245     return if $line =~ m/^\#vzdump\#/;
5246     return if $line =~ m/^lock:/;
5247     return if $line =~ m/^unused\d+:/;
5248     return if $line =~ m/^parent:/;
5249     return if $line =~ m/^template:/; # restored VM is never a template
5250
5251     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5252     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5253         # try to convert old 1.X settings
5254         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5255         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5256             my ($model, $macaddr) = split(/\=/, $devconfig);
5257             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5258             my $net = {
5259                 model => $model,
5260                 bridge => "vmbr$ind",
5261                 macaddr => $macaddr,
5262             };
5263             my $netstr = print_net($net);
5264
5265             print $outfd "net$cookie->{netcount}: $netstr\n";
5266             $cookie->{netcount}++;
5267         }
5268     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5269         my ($id, $netstr) = ($1, $2);
5270         my $net = parse_net($netstr);
5271         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5272         $netstr = print_net($net);
5273         print $outfd "$id: $netstr\n";
5274     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5275         my $virtdev = $1;
5276         my $value = $3;
5277         my $di = parse_drive($virtdev, $value);
5278         if (defined($di->{backup}) && !$di->{backup}) {
5279             print $outfd "#$line";
5280         } elsif ($map->{$virtdev}) {
5281             delete $di->{format}; # format can change on restore
5282             $di->{file} = $map->{$virtdev};
5283             $value = print_drive($vmid, $di);
5284             print $outfd "$virtdev: $value\n";
5285         } else {
5286             print $outfd $line;
5287         }
5288     } else {
5289         print $outfd $line;
5290     }
5291 }
5292
5293 sub scan_volids {
5294     my ($cfg, $vmid) = @_;
5295
5296     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5297
5298     my $volid_hash = {};
5299     foreach my $storeid (keys %$info) {
5300         foreach my $item (@{$info->{$storeid}}) {
5301             next if !($item->{volid} && $item->{size});
5302             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5303             $volid_hash->{$item->{volid}} = $item;
5304         }
5305     }
5306
5307     return $volid_hash;
5308 }
5309
5310 sub is_volume_in_use {
5311     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5312
5313     my $path = PVE::Storage::path($storecfg, $volid);
5314
5315     my $scan_config = sub {
5316         my ($cref, $snapname) = @_;
5317
5318         foreach my $key (keys %$cref) {
5319             my $value = $cref->{$key};
5320             if (is_valid_drivename($key)) {
5321                 next if $skip_drive && $key eq $skip_drive;
5322                 my $drive = parse_drive($key, $value);
5323                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5324                 return 1 if $volid eq $drive->{file};
5325                 if ($drive->{file} =~ m!^/!) {
5326                     return 1 if $drive->{file} eq $path;
5327                 } else {
5328                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5329                     next if !$storeid;
5330                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5331                     next if !$scfg;
5332                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5333                 }
5334             }
5335         }
5336
5337         return 0;
5338     };
5339
5340     return 1 if &$scan_config($conf);
5341
5342     undef $skip_drive;
5343
5344     foreach my $snapname (keys %{$conf->{snapshots}}) {
5345         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5346     }
5347
5348     return 0;
5349 }
5350
5351 sub update_disksize {
5352     my ($vmid, $conf, $volid_hash) = @_;
5353
5354     my $changes;
5355
5356     my $used = {};
5357
5358     # Note: it is allowed to define multiple storages with same path (alias), so
5359     # we need to check both 'volid' and real 'path' (two different volid can point
5360     # to the same path).
5361
5362     my $usedpath = {};
5363
5364     # update size info
5365     foreach my $opt (keys %$conf) {
5366         if (is_valid_drivename($opt)) {
5367             my $drive = parse_drive($opt, $conf->{$opt});
5368             my $volid = $drive->{file};
5369             next if !$volid;
5370
5371             $used->{$volid} = 1;
5372             if ($volid_hash->{$volid} &&
5373                 (my $path = $volid_hash->{$volid}->{path})) {
5374                 $usedpath->{$path} = 1;
5375             }
5376
5377             next if drive_is_cdrom($drive);
5378             next if !$volid_hash->{$volid};
5379
5380             $drive->{size} = $volid_hash->{$volid}->{size};
5381             my $new = print_drive($vmid, $drive);
5382             if ($new ne $conf->{$opt}) {
5383                 $changes = 1;
5384                 $conf->{$opt} = $new;
5385             }
5386         }
5387     }
5388
5389     # remove 'unusedX' entry if volume is used
5390     foreach my $opt (keys %$conf) {
5391         next if $opt !~ m/^unused\d+$/;
5392         my $volid = $conf->{$opt};
5393         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5394         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5395             $changes = 1;
5396             delete $conf->{$opt};
5397         }
5398     }
5399
5400     foreach my $volid (sort keys %$volid_hash) {
5401         next if $volid =~ m/vm-$vmid-state-/;
5402         next if $used->{$volid};
5403         my $path = $volid_hash->{$volid}->{path};
5404         next if !$path; # just to be sure
5405         next if $usedpath->{$path};
5406         $changes = 1;
5407         PVE::QemuConfig->add_unused_volume($conf, $volid);
5408         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5409     }
5410
5411     return $changes;
5412 }
5413
5414 sub rescan {
5415     my ($vmid, $nolock) = @_;
5416
5417     my $cfg = PVE::Storage::config();
5418
5419     my $volid_hash = scan_volids($cfg, $vmid);
5420
5421     my $updatefn =  sub {
5422         my ($vmid) = @_;
5423
5424         my $conf = PVE::QemuConfig->load_config($vmid);
5425
5426         PVE::QemuConfig->check_lock($conf);
5427
5428         my $vm_volids = {};
5429         foreach my $volid (keys %$volid_hash) {
5430             my $info = $volid_hash->{$volid};
5431             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5432         }
5433
5434         my $changes = update_disksize($vmid, $conf, $vm_volids);
5435
5436         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5437     };
5438
5439     if (defined($vmid)) {
5440         if ($nolock) {
5441             &$updatefn($vmid);
5442         } else {
5443             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5444         }
5445     } else {
5446         my $vmlist = config_list();
5447         foreach my $vmid (keys %$vmlist) {
5448             if ($nolock) {
5449                 &$updatefn($vmid);
5450             } else {
5451                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5452             }
5453         }
5454     }
5455 }
5456
5457 sub restore_vma_archive {
5458     my ($archive, $vmid, $user, $opts, $comp) = @_;
5459
5460     my $input = $archive eq '-' ? "<&STDIN" : undef;
5461     my $readfrom = $archive;
5462
5463     my $uncomp = '';
5464     if ($comp) {
5465         $readfrom = '-';
5466         my $qarchive = PVE::Tools::shellquote($archive);
5467         if ($comp eq 'gzip') {
5468             $uncomp = "zcat $qarchive|";
5469         } elsif ($comp eq 'lzop') {
5470             $uncomp = "lzop -d -c $qarchive|";
5471         } else {
5472             die "unknown compression method '$comp'\n";
5473         }
5474
5475     }
5476
5477     my $tmpdir = "/var/tmp/vzdumptmp$$";
5478     rmtree $tmpdir;
5479
5480     # disable interrupts (always do cleanups)
5481     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5482         warn "got interrupt - ignored\n";
5483     };
5484
5485     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5486     POSIX::mkfifo($mapfifo, 0600);
5487     my $fifofh;
5488
5489     my $openfifo = sub {
5490         open($fifofh, '>', $mapfifo) || die $!;
5491     };
5492
5493     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5494
5495     my $oldtimeout;
5496     my $timeout = 5;
5497
5498     my $devinfo = {};
5499
5500     my $rpcenv = PVE::RPCEnvironment::get();
5501
5502     my $conffile = PVE::QemuConfig->config_file($vmid);
5503     my $tmpfn = "$conffile.$$.tmp";
5504
5505     # Note: $oldconf is undef if VM does not exists
5506     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5507     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5508
5509     my $print_devmap = sub {
5510         my $virtdev_hash = {};
5511
5512         my $cfgfn = "$tmpdir/qemu-server.conf";
5513
5514         # we can read the config - that is already extracted
5515         my $fh = IO::File->new($cfgfn, "r") ||
5516             "unable to read qemu-server.conf - $!\n";
5517
5518         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5519         if (-f $fwcfgfn) {
5520             my $pve_firewall_dir = '/etc/pve/firewall';
5521             mkdir $pve_firewall_dir; # make sure the dir exists
5522             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5523         }
5524
5525         while (defined(my $line = <$fh>)) {
5526             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5527                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5528                 die "archive does not contain data for drive '$virtdev'\n"
5529                     if !$devinfo->{$devname};
5530                 if (defined($opts->{storage})) {
5531                     $storeid = $opts->{storage} || 'local';
5532                 } elsif (!$storeid) {
5533                     $storeid = 'local';
5534                 }
5535                 $format = 'raw' if !$format;
5536                 $devinfo->{$devname}->{devname} = $devname;
5537                 $devinfo->{$devname}->{virtdev} = $virtdev;
5538                 $devinfo->{$devname}->{format} = $format;
5539                 $devinfo->{$devname}->{storeid} = $storeid;
5540
5541                 # check permission on storage
5542                 my $pool = $opts->{pool}; # todo: do we need that?
5543                 if ($user ne 'root@pam') {
5544                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5545                 }
5546
5547                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5548             }
5549         }
5550
5551         foreach my $devname (keys %$devinfo) {
5552             die "found no device mapping information for device '$devname'\n"
5553                 if !$devinfo->{$devname}->{virtdev};
5554         }
5555
5556         my $cfg = PVE::Storage::config();
5557
5558         # create empty/temp config
5559         if ($oldconf) {
5560             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5561             foreach_drive($oldconf, sub {
5562                 my ($ds, $drive) = @_;
5563
5564                 return if drive_is_cdrom($drive);
5565
5566                 my $volid = $drive->{file};
5567
5568                 return if !$volid || $volid =~ m|^/|;
5569
5570                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5571                 return if !$path || !$owner || ($owner != $vmid);
5572
5573                 # Note: only delete disk we want to restore
5574                 # other volumes will become unused
5575                 if ($virtdev_hash->{$ds}) {
5576                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5577                     if (my $err = $@) {
5578                         warn $err;
5579                     }
5580                 }
5581             });
5582
5583             # delete vmstate files
5584             # since after the restore we have no snapshots anymore
5585             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5586                 my $snap = $oldconf->{snapshots}->{$snapname};
5587                 if ($snap->{vmstate}) {
5588                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5589                     if (my $err = $@) {
5590                         warn $err;
5591                     }
5592                 }
5593             }
5594         }
5595
5596         my $map = {};
5597         foreach my $virtdev (sort keys %$virtdev_hash) {
5598             my $d = $virtdev_hash->{$virtdev};
5599             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5600             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5601
5602             # test if requested format is supported
5603             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5604             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5605             $d->{format} = $defFormat if !$supported;
5606
5607             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5608                                                   $d->{format}, undef, $alloc_size);
5609             print STDERR "new volume ID is '$volid'\n";
5610             $d->{volid} = $volid;
5611             my $path = PVE::Storage::path($cfg, $volid);
5612
5613             PVE::Storage::activate_volumes($cfg,[$volid]);
5614
5615             my $write_zeros = 1;
5616             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5617                 $write_zeros = 0;
5618             }
5619
5620             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5621
5622             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5623             $map->{$virtdev} = $volid;
5624         }
5625
5626         $fh->seek(0, 0) || die "seek failed - $!\n";
5627
5628         my $outfd = new IO::File ($tmpfn, "w") ||
5629             die "unable to write config for VM $vmid\n";
5630
5631         my $cookie = { netcount => 0 };
5632         while (defined(my $line = <$fh>)) {
5633             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5634         }
5635
5636         $fh->close();
5637         $outfd->close();
5638     };
5639
5640     eval {
5641         # enable interrupts
5642         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5643             die "interrupted by signal\n";
5644         };
5645         local $SIG{ALRM} = sub { die "got timeout\n"; };
5646
5647         $oldtimeout = alarm($timeout);
5648
5649         my $parser = sub {
5650             my $line = shift;
5651
5652             print "$line\n";
5653
5654             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5655                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5656                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5657             } elsif ($line =~ m/^CTIME: /) {
5658                 # we correctly received the vma config, so we can disable
5659                 # the timeout now for disk allocation (set to 10 minutes, so
5660                 # that we always timeout if something goes wrong)
5661                 alarm(600);
5662                 &$print_devmap();
5663                 print $fifofh "done\n";
5664                 my $tmp = $oldtimeout || 0;
5665                 $oldtimeout = undef;
5666                 alarm($tmp);
5667                 close($fifofh);
5668             }
5669         };
5670
5671         print "restore vma archive: $cmd\n";
5672         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5673     };
5674     my $err = $@;
5675
5676     alarm($oldtimeout) if $oldtimeout;
5677
5678     my $vollist = [];
5679     foreach my $devname (keys %$devinfo) {
5680         my $volid = $devinfo->{$devname}->{volid};
5681         push @$vollist, $volid if $volid;
5682     }
5683
5684     my $cfg = PVE::Storage::config();
5685     PVE::Storage::deactivate_volumes($cfg, $vollist);
5686
5687     unlink $mapfifo;
5688
5689     if ($err) {
5690         rmtree $tmpdir;
5691         unlink $tmpfn;
5692
5693         foreach my $devname (keys %$devinfo) {
5694             my $volid = $devinfo->{$devname}->{volid};
5695             next if !$volid;
5696             eval {
5697                 if ($volid =~ m|^/|) {
5698                     unlink $volid || die 'unlink failed\n';
5699                 } else {
5700                     PVE::Storage::vdisk_free($cfg, $volid);
5701                 }
5702                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5703             };
5704             print STDERR "unable to cleanup '$volid' - $@" if $@;
5705         }
5706         die $err;
5707     }
5708
5709     rmtree $tmpdir;
5710
5711     rename($tmpfn, $conffile) ||
5712         die "unable to commit configuration file '$conffile'\n";
5713
5714     PVE::Cluster::cfs_update(); # make sure we read new file
5715
5716     eval { rescan($vmid, 1); };
5717     warn $@ if $@;
5718 }
5719
5720 sub restore_tar_archive {
5721     my ($archive, $vmid, $user, $opts) = @_;
5722
5723     if ($archive ne '-') {
5724         my $firstfile = tar_archive_read_firstfile($archive);
5725         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5726             if $firstfile ne 'qemu-server.conf';
5727     }
5728
5729     my $storecfg = PVE::Storage::config();
5730
5731     # destroy existing data - keep empty config
5732     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5733     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5734
5735     my $tocmd = "/usr/lib/qemu-server/qmextract";
5736
5737     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5738     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5739     $tocmd .= ' --prealloc' if $opts->{prealloc};
5740     $tocmd .= ' --info' if $opts->{info};
5741
5742     # tar option "xf" does not autodetect compression when read from STDIN,
5743     # so we pipe to zcat
5744     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5745         PVE::Tools::shellquote("--to-command=$tocmd");
5746
5747     my $tmpdir = "/var/tmp/vzdumptmp$$";
5748     mkpath $tmpdir;
5749
5750     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5751     local $ENV{VZDUMP_VMID} = $vmid;
5752     local $ENV{VZDUMP_USER} = $user;
5753
5754     my $conffile = PVE::QemuConfig->config_file($vmid);
5755     my $tmpfn = "$conffile.$$.tmp";
5756
5757     # disable interrupts (always do cleanups)
5758     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5759         print STDERR "got interrupt - ignored\n";
5760     };
5761
5762     eval {
5763         # enable interrupts
5764         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5765             die "interrupted by signal\n";
5766         };
5767
5768         if ($archive eq '-') {
5769             print "extracting archive from STDIN\n";
5770             run_command($cmd, input => "<&STDIN");
5771         } else {
5772             print "extracting archive '$archive'\n";
5773             run_command($cmd);
5774         }
5775
5776         return if $opts->{info};
5777
5778         # read new mapping
5779         my $map = {};
5780         my $statfile = "$tmpdir/qmrestore.stat";
5781         if (my $fd = IO::File->new($statfile, "r")) {
5782             while (defined (my $line = <$fd>)) {
5783                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5784                     $map->{$1} = $2 if $1;
5785                 } else {
5786                     print STDERR "unable to parse line in statfile - $line\n";
5787                 }
5788             }
5789             $fd->close();
5790         }
5791
5792         my $confsrc = "$tmpdir/qemu-server.conf";
5793
5794         my $srcfd = new IO::File($confsrc, "r") ||
5795             die "unable to open file '$confsrc'\n";
5796
5797         my $outfd = new IO::File ($tmpfn, "w") ||
5798             die "unable to write config for VM $vmid\n";
5799
5800         my $cookie = { netcount => 0 };
5801         while (defined (my $line = <$srcfd>)) {
5802             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5803         }
5804
5805         $srcfd->close();
5806         $outfd->close();
5807     };
5808     my $err = $@;
5809
5810     if ($err) {
5811
5812         unlink $tmpfn;
5813
5814         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5815
5816         die $err;
5817     }
5818
5819     rmtree $tmpdir;
5820
5821     rename $tmpfn, $conffile ||
5822         die "unable to commit configuration file '$conffile'\n";
5823
5824     PVE::Cluster::cfs_update(); # make sure we read new file
5825
5826     eval { rescan($vmid, 1); };
5827     warn $@ if $@;
5828 };
5829
5830 sub foreach_writable_storage {
5831     my ($conf, $func) = @_;
5832
5833     my $sidhash = {};
5834
5835     foreach my $ds (keys %$conf) {
5836         next if !is_valid_drivename($ds);
5837
5838         my $drive = parse_drive($ds, $conf->{$ds});
5839         next if !$drive;
5840         next if drive_is_cdrom($drive);
5841
5842         my $volid = $drive->{file};
5843
5844         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5845         $sidhash->{$sid} = $sid if $sid;
5846     }
5847
5848     foreach my $sid (sort keys %$sidhash) {
5849         &$func($sid);
5850     }
5851 }
5852
5853 sub do_snapshots_with_qemu {
5854     my ($storecfg, $volid) = @_;
5855
5856     my $storage_name = PVE::Storage::parse_volume_id($volid);
5857
5858     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5859         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5860         return 1;
5861     }
5862
5863     if ($volid =~ m/\.(qcow2|qed)$/){
5864         return 1;
5865     }
5866
5867     return undef;
5868 }
5869
5870 sub qga_check_running {
5871     my ($vmid) = @_;
5872
5873     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5874     if ($@) {
5875         warn "Qemu Guest Agent are not running - $@";
5876         return 0;
5877     }
5878     return 1;
5879 }
5880
5881 sub template_create {
5882     my ($vmid, $conf, $disk) = @_;
5883
5884     my $storecfg = PVE::Storage::config();
5885
5886     foreach_drive($conf, sub {
5887         my ($ds, $drive) = @_;
5888
5889         return if drive_is_cdrom($drive);
5890         return if $disk && $ds ne $disk;
5891
5892         my $volid = $drive->{file};
5893         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5894
5895         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5896         $drive->{file} = $voliddst;
5897         $conf->{$ds} = print_drive($vmid, $drive);
5898         PVE::QemuConfig->write_config($vmid, $conf);
5899     });
5900 }
5901
5902 sub qemu_img_convert {
5903     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5904
5905     my $storecfg = PVE::Storage::config();
5906     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5907     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5908
5909     if ($src_storeid && $dst_storeid) {
5910
5911         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5912
5913         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5914         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5915
5916         my $src_format = qemu_img_format($src_scfg, $src_volname);
5917         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5918
5919         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5920         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5921
5922         my $cmd = [];
5923         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5924         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5925         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5926         if ($is_zero_initialized) {
5927             push @$cmd, "zeroinit:$dst_path";
5928         } else {
5929             push @$cmd, $dst_path;
5930         }
5931
5932         my $parser = sub {
5933             my $line = shift;
5934             if($line =~ m/\((\S+)\/100\%\)/){
5935                 my $percent = $1;
5936                 my $transferred = int($size * $percent / 100);
5937                 my $remaining = $size - $transferred;
5938
5939                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5940             }
5941
5942         };
5943
5944         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5945         my $err = $@;
5946         die "copy failed: $err" if $err;
5947     }
5948 }
5949
5950 sub qemu_img_format {
5951     my ($scfg, $volname) = @_;
5952
5953     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5954         return $1;
5955     } else {
5956         return "raw";
5957     }
5958 }
5959
5960 sub qemu_drive_mirror {
5961     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
5962
5963     $jobs = {} if !$jobs;
5964
5965     my $qemu_target;
5966     my $format;
5967     $jobs->{"drive-$drive"} = {};
5968
5969     if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
5970         my $server = $1;
5971         my $port = $2;
5972         my $exportname = $3;
5973
5974         $format = "nbd";
5975         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
5976         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
5977         my $cmd = ['socat', '-T30', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=5"];
5978
5979         my $pid = fork();
5980         if (!defined($pid)) {
5981             die "forking socat tunnel failed\n";
5982         } elsif ($pid == 0) {
5983             exec(@$cmd);
5984             warn "exec failed: $!\n";
5985             POSIX::_exit(-1);
5986         }
5987         $jobs->{"drive-$drive"}->{pid} = $pid;
5988
5989         my $timeout = 0;
5990         while (!-S $unixsocket) {
5991             die "nbd connection helper timed out\n"
5992                 if $timeout++ > 5;
5993             sleep 1;
5994         }
5995     } else {
5996         my $storecfg = PVE::Storage::config();
5997         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5998
5999         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6000
6001         $format = qemu_img_format($dst_scfg, $dst_volname);
6002
6003         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6004
6005         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6006     }
6007
6008     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6009     $opts->{format} = $format if $format;
6010
6011     print "drive mirror is starting for drive-$drive\n";
6012
6013     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6014
6015     if (my $err = $@) {
6016         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6017         die "mirroring error: $err";
6018     }
6019
6020     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6021 }
6022
6023 sub qemu_drive_mirror_monitor {
6024     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6025
6026     eval {
6027         my $err_complete = 0;
6028
6029         while (1) {
6030             die "storage migration timed out\n" if $err_complete > 300;
6031
6032             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6033
6034             my $running_mirror_jobs = {};
6035             foreach my $stat (@$stats) {
6036                 next if $stat->{type} ne 'mirror';
6037                 $running_mirror_jobs->{$stat->{device}} = $stat;
6038             }
6039
6040             my $readycounter = 0;
6041
6042             foreach my $job (keys %$jobs) {
6043
6044                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6045                     print "$job : finished\n";
6046                     delete $jobs->{$job};
6047                     next;
6048                 }
6049
6050                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6051
6052                 my $busy = $running_mirror_jobs->{$job}->{busy};
6053                 my $ready = $running_mirror_jobs->{$job}->{ready};
6054                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6055                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6056                     my $remaining = $total - $transferred;
6057                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6058
6059                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6060                 }
6061
6062                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6063             }
6064
6065             last if scalar(keys %$jobs) == 0;
6066
6067             if ($readycounter == scalar(keys %$jobs)) {
6068                 print "all mirroring jobs are ready \n";
6069                 last if $skipcomplete; #do the complete later
6070
6071                 if ($vmiddst && $vmiddst != $vmid) {
6072                     if ($qga) {
6073                         print "freeze filesystem\n";
6074                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6075                     } else {
6076                         print "suspend vm\n";
6077                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6078                     }
6079
6080                     # if we clone a disk for a new target vm, we don't switch the disk
6081                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6082
6083                     if ($qga) {
6084                         print "unfreeze filesystem\n";
6085                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6086                     } else {
6087                         print "resume vm\n";
6088                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6089                     }
6090
6091                     last;
6092                 } else {
6093
6094                     foreach my $job (keys %$jobs) {
6095                         # try to switch the disk if source and destination are on the same guest
6096                         print "$job: Completing block job...\n";
6097
6098                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6099                         if ($@ =~ m/cannot be completed/) {
6100                             print "$job: Block job cannot be completed, try again.\n";
6101                             $err_complete++;
6102                         }else {
6103                             print "$job: Completed successfully.\n";
6104                             $jobs->{$job}->{complete} = 1;
6105                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6106                         }
6107                     }
6108                 }
6109             }
6110             sleep 1;
6111         }
6112     };
6113     my $err = $@;
6114
6115     if ($err) {
6116         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6117         die "mirroring error: $err";
6118     }
6119
6120 }
6121
6122 sub qemu_blockjobs_cancel {
6123     my ($vmid, $jobs) = @_;
6124
6125     foreach my $job (keys %$jobs) {
6126         print "$job: Cancelling block job\n";
6127         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6128         $jobs->{$job}->{cancel} = 1;
6129     }
6130
6131     while (1) {
6132         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6133
6134         my $running_jobs = {};
6135         foreach my $stat (@$stats) {
6136             $running_jobs->{$stat->{device}} = $stat;
6137         }
6138
6139         foreach my $job (keys %$jobs) {
6140
6141             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6142                 print "$job: Done.\n";
6143                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6144                 delete $jobs->{$job};
6145             }
6146         }
6147
6148         last if scalar(keys %$jobs) == 0;
6149
6150         sleep 1;
6151     }
6152 }
6153
6154 sub qemu_blockjobs_finish_tunnel {
6155    my ($vmid, $job, $cpid) = @_;
6156
6157    return if !$cpid;
6158
6159    for (my $i = 1; $i < 20; $i++) {
6160         my $waitpid = waitpid($cpid, WNOHANG);
6161         last if (defined($waitpid) && ($waitpid == $cpid));
6162
6163         if ($i == 10) {
6164             kill(15, $cpid);
6165         } elsif ($i >= 15) {
6166             kill(9, $cpid);
6167         }
6168         sleep (1);
6169     }
6170     unlink "/run/qemu-server/$vmid.mirror-$job";
6171 }
6172
6173 sub clone_disk {
6174     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6175         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6176
6177     my $newvolid;
6178
6179     if (!$full) {
6180         print "create linked clone of drive $drivename ($drive->{file})\n";
6181         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6182         push @$newvollist, $newvolid;
6183     } else {
6184
6185         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6186         $storeid = $storage if $storage;
6187
6188         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6189         if (!$format) {
6190             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6191             $format = qemu_img_format($scfg, $volname);
6192         }
6193
6194         # test if requested format is supported - else use default
6195         my $supported = grep { $_ eq $format } @$validFormats;
6196         $format = $defFormat if !$supported;
6197
6198         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6199
6200         print "create full clone of drive $drivename ($drive->{file})\n";
6201         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6202         push @$newvollist, $newvolid;
6203
6204         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6205
6206         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6207         if (!$running || $snapname) {
6208             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6209         } else {
6210
6211             my $kvmver = get_running_qemu_version ($vmid);
6212             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6213                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6214                     if $drive->{iothread};
6215             }
6216
6217             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6218         }
6219     }
6220
6221     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6222
6223     my $disk = $drive;
6224     $disk->{format} = undef;
6225     $disk->{file} = $newvolid;
6226     $disk->{size} = $size;
6227
6228     return $disk;
6229 }
6230
6231 # this only works if VM is running
6232 sub get_current_qemu_machine {
6233     my ($vmid) = @_;
6234
6235     my $cmd = { execute => 'query-machines', arguments => {} };
6236     my $res = vm_qmp_command($vmid, $cmd);
6237
6238     my ($current, $default);
6239     foreach my $e (@$res) {
6240         $default = $e->{name} if $e->{'is-default'};
6241         $current = $e->{name} if $e->{'is-current'};
6242     }
6243
6244     # fallback to the default machine if current is not supported by qemu
6245     return $current || $default || 'pc';
6246 }
6247
6248 sub get_running_qemu_version {
6249     my ($vmid) = @_;
6250     my $cmd = { execute => 'query-version', arguments => {} };
6251     my $res = vm_qmp_command($vmid, $cmd);
6252     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6253 }
6254
6255 sub qemu_machine_feature_enabled {
6256     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6257
6258     my $current_major;
6259     my $current_minor;
6260
6261     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6262
6263         $current_major = $3;
6264         $current_minor = $4;
6265
6266     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6267
6268         $current_major = $1;
6269         $current_minor = $2;
6270     }
6271
6272     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6273
6274
6275 }
6276
6277 sub qemu_machine_pxe {
6278     my ($vmid, $conf, $machine) = @_;
6279
6280     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6281
6282     foreach my $opt (keys %$conf) {
6283         next if $opt !~ m/^net(\d+)$/;
6284         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6285         next if !$net;
6286         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6287         return $machine.".pxe" if $romfile =~ m/pxe/;
6288         last;
6289     }
6290
6291     return $machine;
6292 }
6293
6294 sub qemu_use_old_bios_files {
6295     my ($machine_type) = @_;
6296
6297     return if !$machine_type;
6298
6299     my $use_old_bios_files = undef;
6300
6301     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6302         $machine_type = $1;
6303         $use_old_bios_files = 1;
6304     } else {
6305         my $kvmver = kvm_user_version();
6306         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6307         # load new efi bios files on migration. So this hack is required to allow
6308         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6309         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6310         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6311     }
6312
6313     return ($use_old_bios_files, $machine_type);
6314 }
6315
6316 sub lspci {
6317
6318     my $devices = {};
6319
6320     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6321             my (undef, $id, $function) = @_;
6322             my $res = { id => $id, function => $function};
6323             push @{$devices->{$id}}, $res;
6324     });
6325
6326     return $devices;
6327 }
6328
6329 sub vm_iothreads_list {
6330     my ($vmid) = @_;
6331
6332     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6333
6334     my $iothreads = {};
6335     foreach my $iothread (@$res) {
6336         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6337     }
6338
6339     return $iothreads;
6340 }
6341
6342 sub scsihw_infos {
6343     my ($conf, $drive) = @_;
6344
6345     my $maxdev = 0;
6346
6347     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6348         $maxdev = 7;
6349     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6350         $maxdev = 1;
6351     } else {
6352         $maxdev = 256;
6353     }
6354
6355     my $controller = int($drive->{index} / $maxdev);
6356     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6357
6358     return ($maxdev, $controller, $controller_prefix);
6359 }
6360
6361 sub add_hyperv_enlighments {
6362     my ($cpuFlags, $winversion, $machine_type, $kvmver, $nokvm, $bios, $gpu_passthrough) = @_;
6363
6364     return if $nokvm;
6365     return if $winversion < 6;
6366     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6367
6368     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6369
6370     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6371         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6372         push @$cpuFlags , 'hv_vapic';
6373         push @$cpuFlags , 'hv_time';
6374     } else {
6375         push @$cpuFlags , 'hv_spinlocks=0xffff';
6376     }
6377
6378     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6379         push @$cpuFlags , 'hv_reset';
6380         push @$cpuFlags , 'hv_vpindex';
6381         push @$cpuFlags , 'hv_runtime';
6382     }
6383
6384     if ($winversion >= 7) {
6385         push @$cpuFlags , 'hv_relaxed';
6386     }
6387 }
6388
6389 sub windows_version {
6390     my ($ostype) = @_;
6391
6392     return 0 if !$ostype;
6393
6394     my $winversion = 0;
6395
6396     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6397         $winversion = 5;
6398     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6399         $winversion = 6;
6400     } elsif ($ostype =~ m/^win(\d+)$/) {
6401         $winversion = $1;
6402     }
6403
6404     return $winversion;
6405 }
6406
6407 # bash completion helper
6408
6409 sub complete_backup_archives {
6410     my ($cmdname, $pname, $cvalue) = @_;
6411
6412     my $cfg = PVE::Storage::config();
6413
6414     my $storeid;
6415
6416     if ($cvalue =~ m/^([^:]+):/) {
6417         $storeid = $1;
6418     }
6419
6420     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6421
6422     my $res = [];
6423     foreach my $id (keys %$data) {
6424         foreach my $item (@{$data->{$id}}) {
6425             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6426             push @$res, $item->{volid} if defined($item->{volid});
6427         }
6428     }
6429
6430     return $res;
6431 }
6432
6433 my $complete_vmid_full = sub {
6434     my ($running) = @_;
6435
6436     my $idlist = vmstatus();
6437
6438     my $res = [];
6439
6440     foreach my $id (keys %$idlist) {
6441         my $d = $idlist->{$id};
6442         if (defined($running)) {
6443             next if $d->{template};
6444             next if $running && $d->{status} ne 'running';
6445             next if !$running && $d->{status} eq 'running';
6446         }
6447         push @$res, $id;
6448
6449     }
6450     return $res;
6451 };
6452
6453 sub complete_vmid {
6454     return &$complete_vmid_full();
6455 }
6456
6457 sub complete_vmid_stopped {
6458     return &$complete_vmid_full(0);
6459 }
6460
6461 sub complete_vmid_running {
6462     return &$complete_vmid_full(1);
6463 }
6464
6465 sub complete_storage {
6466
6467     my $cfg = PVE::Storage::config();
6468     my $ids = $cfg->{ids};
6469
6470     my $res = [];
6471     foreach my $sid (keys %$ids) {
6472         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6473         next if !$ids->{$sid}->{content}->{images};
6474         push @$res, $sid;
6475     }
6476
6477     return $res;
6478 }
6479
6480 sub nbd_stop {
6481     my ($vmid) = @_;
6482
6483     vm_mon_cmd($vmid, 'nbd-server-stop');
6484 }
6485
6486 1;