PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42 my $OVMF_IMG = '/usr/share/kvm/OVMF-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 # Note about locking: we use flock on the config file protect
  49 # against concurent actions.
  50 # Aditionaly, we have a 'lock' setting in the config file. This
  51 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  52 # allowed when such lock is set. But you can ignore this kind of
  53 # lock with the --skiplock flag.
  54
  55 cfs_register_file('/qemu-server/',
  56                   \&parse_vm_config,
  57                   \&write_vm_config);
  58
  59 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  60     description => "Some command save/restore state from this location.",
  61     type => 'string',
  62     maxLength => 128,
  63     optional => 1,
  64 });
  65
  66 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  67     description => "The name of the snapshot.",
  68     type => 'string', format => 'pve-configid',
  69     maxLength => 40,
  70 });
  71
  72 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  73     type => 'string',
  74     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  75     description => "The drive's backing file's data format.",
  76     optional => 1,
  77 });
  78
  79 #no warnings 'redefine';
  80
  81 sub cgroups_write {
  82    my ($controller, $vmid, $option, $value) = @_;
  83
  84    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  85    PVE::ProcFSTools::write_proc_entry($path, $value);
  86
  87 }
  88
  89 my $nodename = PVE::INotify::nodename();
  90
  91 mkdir "/etc/pve/nodes/$nodename";
  92 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  93 mkdir $confdir;
  94
  95 my $var_run_tmpdir = "/var/run/qemu-server";
  96 mkdir $var_run_tmpdir;
  97
  98 my $lock_dir = "/var/lock/qemu-server";
  99 mkdir $lock_dir;
 100
 101 my $pcisysfs = "/sys/bus/pci";
 102
 103 my $cpu_vendor_list = {
 104     # Intel CPUs
 105     486 => 'GenuineIntel',
 106     pentium => 'GenuineIntel',
 107     pentium2  => 'GenuineIntel',
 108     pentium3  => 'GenuineIntel',
 109     coreduo => 'GenuineIntel',
 110     core2duo => 'GenuineIntel',
 111     Conroe  => 'GenuineIntel',
 112     Penryn  => 'GenuineIntel',
 113     Nehalem  => 'GenuineIntel',
 114     Westmere => 'GenuineIntel',
 115     SandyBridge => 'GenuineIntel',
 116     IvyBridge => 'GenuineIntel',
 117     Haswell => 'GenuineIntel',
 118     'Haswell-noTSX' => 'GenuineIntel',
 119     Broadwell => 'GenuineIntel',
 120     'Broadwell-noTSX' => 'GenuineIntel',
 121     'Skylake-Client' => 'GenuineIntel',
 122
 123     # AMD CPUs
 124     athlon => 'AuthenticAMD',
 125     phenom  => 'AuthenticAMD',
 126     Opteron_G1  => 'AuthenticAMD',
 127     Opteron_G2  => 'AuthenticAMD',
 128     Opteron_G3  => 'AuthenticAMD',
 129     Opteron_G4  => 'AuthenticAMD',
 130     Opteron_G5  => 'AuthenticAMD',
 131
 132     # generic types, use vendor from host node
 133     host => 'default',
 134     kvm32 => 'default',
 135     kvm64 => 'default',
 136     qemu32 => 'default',
 137     qemu64 => 'default',
 138 };
 139
 140 my $cpu_fmt = {
 141     cputype => {
 142         description => "Emulated CPU type.",
 143         type => 'string',
 144         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 145         default => 'kvm64',
 146         default_key => 1,
 147     },
 148     hidden => {
 149         description => "Do not identify as a KVM virtual machine.",
 150         type => 'boolean',
 151         optional => 1,
 152         default => 0
 153     },
 154 };
 155
 156 my $watchdog_fmt = {
 157     model => {
 158         default_key => 1,
 159         type => 'string',
 160         enum => [qw(i6300esb ib700)],
 161         description => "Watchdog type to emulate.",
 162         default => 'i6300esb',
 163         optional => 1,
 164     },
 165     action => {
 166         type => 'string',
 167         enum => [qw(reset shutdown poweroff pause debug none)],
 168         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 169         optional => 1,
 170     },
 171 };
 172 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 173
 174 my $confdesc = {
 175     onboot => {
 176         optional => 1,
 177         type => 'boolean',
 178         description => "Specifies whether a VM will be started during system bootup.",
 179         default => 0,
 180     },
 181     autostart => {
 182         optional => 1,
 183         type => 'boolean',
 184         description => "Automatic restart after crash (currently ignored).",
 185         default => 0,
 186     },
 187     hotplug => {
 188         optional => 1,
 189         type => 'string', format => 'pve-hotplug-features',
 190         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 191         default => 'network,disk,usb',
 192     },
 193     reboot => {
 194         optional => 1,
 195         type => 'boolean',
 196         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 197         default => 1,
 198     },
 199     lock => {
 200         optional => 1,
 201         type => 'string',
 202         description => "Lock/unlock the VM.",
 203         enum => [qw(migrate backup snapshot rollback)],
 204     },
 205     cpulimit => {
 206         optional => 1,
 207         type => 'number',
 208         description => "Limit of CPU usage.",
 209         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 210         minimum => 0,
 211         maximum => 128,
 212         default => 0,
 213     },
 214     cpuunits => {
 215         optional => 1,
 216         type => 'integer',
 217         description => "CPU weight for a VM.",
 218         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 219         minimum => 0,
 220         maximum => 500000,
 221         default => 1024,
 222     },
 223     memory => {
 224         optional => 1,
 225         type => 'integer',
 226         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 227         minimum => 16,
 228         default => 512,
 229     },
 230     balloon => {
 231         optional => 1,
 232         type => 'integer',
 233         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 234         minimum => 0,
 235     },
 236     shares => {
 237         optional => 1,
 238         type => 'integer',
 239         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 240         minimum => 0,
 241         maximum => 50000,
 242         default => 1000,
 243     },
 244     keyboard => {
 245         optional => 1,
 246         type => 'string',
 247         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 248         enum => PVE::Tools::kvmkeymaplist(),
 249         default => 'en-us',
 250     },
 251     name => {
 252         optional => 1,
 253         type => 'string', format => 'dns-name',
 254         description => "Set a name for the VM. Only used on the configuration web interface.",
 255     },
 256     scsihw => {
 257         optional => 1,
 258         type => 'string',
 259         description => "SCSI controller model",
 260         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 261         default => 'lsi',
 262     },
 263     description => {
 264         optional => 1,
 265         type => 'string',
 266         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 267     },
 268     ostype => {
 269         optional => 1,
 270         type => 'string',
 271         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 272         description => "Specify guest operating system.",
 273         verbose_description => <<EODESC,
 274 Specify guest operating system. This is used to enable special
 275 optimization/features for specific operating systems:
 276
 277 [horizontal]
 278 other;; unspecified OS
 279 wxp;; Microsoft Windows XP
 280 w2k;; Microsoft Windows 2000
 281 w2k3;; Microsoft Windows 2003
 282 w2k8;; Microsoft Windows 2008
 283 wvista;; Microsoft Windows Vista
 284 win7;; Microsoft Windows 7
 285 win8;; Microsoft Windows 8/2012
 286 l24;; Linux 2.4 Kernel
 287 l26;; Linux 2.6/3.X Kernel
 288 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 289 EODESC
 290     },
 291     boot => {
 292         optional => 1,
 293         type => 'string',
 294         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 295         pattern => '[acdn]{1,4}',
 296         default => 'cdn',
 297     },
 298     bootdisk => {
 299         optional => 1,
 300         type => 'string', format => 'pve-qm-bootdisk',
 301         description => "Enable booting from specified disk.",
 302         pattern => '(ide|sata|scsi|virtio)\d+',
 303     },
 304     smp => {
 305         optional => 1,
 306         type => 'integer',
 307         description => "The number of CPUs. Please use option -sockets instead.",
 308         minimum => 1,
 309         default => 1,
 310     },
 311     sockets => {
 312         optional => 1,
 313         type => 'integer',
 314         description => "The number of CPU sockets.",
 315         minimum => 1,
 316         default => 1,
 317     },
 318     cores => {
 319         optional => 1,
 320         type => 'integer',
 321         description => "The number of cores per socket.",
 322         minimum => 1,
 323         default => 1,
 324     },
 325     numa => {
 326         optional => 1,
 327         type => 'boolean',
 328         description => "Enable/disable NUMA.",
 329         default => 0,
 330     },
 331     hugepages => {
 332         optional => 1,
 333         type => 'string',
 334         description => "Enable/disable hugepages memory.",
 335         enum => [qw(any 2 1024)],
 336     },
 337     vcpus => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "Number of hotplugged vcpus.",
 341         minimum => 1,
 342         default => 0,
 343     },
 344     acpi => {
 345         optional => 1,
 346         type => 'boolean',
 347         description => "Enable/disable ACPI.",
 348         default => 1,
 349     },
 350     agent => {
 351         optional => 1,
 352         type => 'boolean',
 353         description => "Enable/disable Qemu GuestAgent.",
 354         default => 0,
 355     },
 356     kvm => {
 357         optional => 1,
 358         type => 'boolean',
 359         description => "Enable/disable KVM hardware virtualization.",
 360         default => 1,
 361     },
 362     tdf => {
 363         optional => 1,
 364         type => 'boolean',
 365         description => "Enable/disable time drift fix.",
 366         default => 0,
 367     },
 368     localtime => {
 369         optional => 1,
 370         type => 'boolean',
 371         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 372     },
 373     freeze => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 377     },
 378     vga => {
 379         optional => 1,
 380         type => 'string',
 381         description => "Select the VGA type.",
 382         verbose_description => "Select the VGA type. If you want to use high resolution" .
 383             " modes (>= 1280x1024x16) then you should use the options " .
 384             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 385             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 386             "display sever. For win* OS you can select how many independent " .
 387             "displays you want, Linux guests can add displays them self. " .
 388             "You can also run without any graphic card, using a serial device" .
 389             " as terminal.",
 390         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 391     },
 392     watchdog => {
 393         optional => 1,
 394         type => 'string', format => 'pve-qm-watchdog',
 395         description => "Create a virtual hardware watchdog device.",
 396         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 397             " (by a guest action), the watchdog must be periodically polled " .
 398             "by an agent inside the guest or else the watchdog will reset " .
 399             "the guest (or execute the respective action specified)",
 400     },
 401     startdate => {
 402         optional => 1,
 403         type => 'string',
 404         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 405         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 406         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 407         default => 'now',
 408     },
 409     startup =>  get_standard_option('pve-startup-order'),
 410     template => {
 411         optional => 1,
 412         type => 'boolean',
 413         description => "Enable/disable Template.",
 414         default => 0,
 415     },
 416     args => {
 417         optional => 1,
 418         type => 'string',
 419         description => "Arbitrary arguments passed to kvm.",
 420         verbose_description => <<EODESCR,
 421 Arbitrary arguments passed to kvm, for example:
 422
 423 args: -no-reboot -no-hpet
 424
 425 NOTE: this option is for experts only.
 426 EODESCR
 427     },
 428     tablet => {
 429         optional => 1,
 430         type => 'boolean',
 431         default => 1,
 432         description => "Enable/disable the USB tablet device.",
 433         verbose_description => "Enable/disable the USB tablet device. This device is " .
 434             "usually needed to allow absolute mouse positioning with VNC. " .
 435             "Else the mouse runs out of sync with normal VNC clients. " .
 436             "If you're running lots of console-only guests on one host, " .
 437             "you may consider disabling this to save some context switches. " .
 438             "This is turned off by default if you use spice (-vga=qxl).",
 439     },
 440     migrate_speed => {
 441         optional => 1,
 442         type => 'integer',
 443         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 444         minimum => 0,
 445         default => 0,
 446     },
 447     migrate_downtime => {
 448         optional => 1,
 449         type => 'number',
 450         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 451         minimum => 0,
 452         default => 0.1,
 453     },
 454     cdrom => {
 455         optional => 1,
 456         type => 'string', format => 'pve-qm-ide',
 457         typetext => '<volume>',
 458         description => "This is an alias for option -ide2",
 459     },
 460     cpu => {
 461         optional => 1,
 462         description => "Emulated CPU type.",
 463         type => 'string',
 464         format => $cpu_fmt,
 465     },
 466     parent => get_standard_option('pve-snapshot-name', {
 467         optional => 1,
 468         description => "Parent snapshot name. This is used internally, and should not be modified.",
 469     }),
 470     snaptime => {
 471         optional => 1,
 472         description => "Timestamp for snapshots.",
 473         type => 'integer',
 474         minimum => 0,
 475     },
 476     vmstate => {
 477         optional => 1,
 478         type => 'string', format => 'pve-volume-id',
 479         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 480     },
 481     machine => {
 482         description => "Specific the Qemu machine type.",
 483         type => 'string',
 484         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 485         maxLength => 40,
 486         optional => 1,
 487     },
 488     smbios1 => {
 489         description => "Specify SMBIOS type 1 fields.",
 490         type => 'string', format => 'pve-qm-smbios1',
 491         maxLength => 256,
 492         optional => 1,
 493     },
 494     replicate => get_standard_option('pve-replicate'),
 495     protection => {
 496         optional => 1,
 497         type => 'boolean',
 498         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 499         default => 0,
 500     },
 501     bios => {
 502         optional => 1,
 503         type => 'string',
 504         enum => [ qw(seabios ovmf) ],
 505         description => "Select BIOS implementation.",
 506         default => 'seabios',
 507     },
 508 };
 509
 510 # what about other qemu settings ?
 511 #cpu => 'string',
 512 #machine => 'string',
 513 #fda => 'file',
 514 #fdb => 'file',
 515 #mtdblock => 'file',
 516 #sd => 'file',
 517 #pflash => 'file',
 518 #snapshot => 'bool',
 519 #bootp => 'file',
 520 ##tftp => 'dir',
 521 ##smb => 'dir',
 522 #kernel => 'file',
 523 #append => 'string',
 524 #initrd => 'file',
 525 ##soundhw => 'string',
 526
 527 while (my ($k, $v) = each %$confdesc) {
 528     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 529 }
 530
 531 my $MAX_IDE_DISKS = 4;
 532 my $MAX_SCSI_DISKS = 14;
 533 my $MAX_VIRTIO_DISKS = 16;
 534 my $MAX_SATA_DISKS = 6;
 535 my $MAX_USB_DEVICES = 5;
 536 my $MAX_NETS = 32;
 537 my $MAX_UNUSED_DISKS = 8;
 538 my $MAX_HOSTPCI_DEVICES = 4;
 539 my $MAX_SERIAL_PORTS = 4;
 540 my $MAX_PARALLEL_PORTS = 3;
 541 my $MAX_NUMA = 8;
 542
 543 my $numa_fmt = {
 544     cpus => {
 545         type => "string",
 546         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 547         description => "CPUs accessing this NUMA node.",
 548         format_description => "id[-id];...",
 549     },
 550     memory => {
 551         type => "number",
 552         description => "Amount of memory this NUMA node provides.",
 553         optional => 1,
 554     },
 555     hostnodes => {
 556         type => "string",
 557         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 558         description => "Host NUMA nodes to use.",
 559         format_description => "id[-id];...",
 560         optional => 1,
 561     },
 562     policy => {
 563         type => 'string',
 564         enum => [qw(preferred bind interleave)],
 565         description => "NUMA allocation policy.",
 566         optional => 1,
 567     },
 568 };
 569 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 570 my $numadesc = {
 571     optional => 1,
 572     type => 'string', format => $numa_fmt,
 573     description => "NUMA topology.",
 574 };
 575 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 576
 577 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 578     $confdesc->{"numa$i"} = $numadesc;
 579 }
 580
 581 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 582                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 583                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 584 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 585
 586 my $net_fmt_bridge_descr = <<__EOD__;
 587 Bridge to attach the network device to. The Proxmox VE standard bridge
 588 is called 'vmbr0'.
 589
 590 If you do not specify a bridge, we create a kvm user (NATed) network
 591 device, which provides DHCP and DNS services. The following addresses
 592 are used:
 593
 594  10.0.2.2   Gateway
 595  10.0.2.3   DNS Server
 596  10.0.2.4   SMB Server
 597
 598 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 599 __EOD__
 600
 601 my $net_fmt = {
 602     macaddr => {
 603         type => 'string',
 604         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 605         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 606         format_description => "XX:XX:XX:XX:XX:XX",
 607         optional => 1,
 608     },
 609     model => {
 610         type => 'string',
 611         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 612         enum => $nic_model_list,
 613         default_key => 1,
 614     },
 615     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 616     bridge => {
 617         type => 'string',
 618         description => $net_fmt_bridge_descr,
 619         format_description => 'bridge',
 620         optional => 1,
 621     },
 622     queues => {
 623         type => 'integer',
 624         minimum => 0, maximum => 16,
 625         description => 'Number of packet queues to be used on the device.',
 626         optional => 1,
 627     },
 628     rate => {
 629         type => 'number',
 630         minimum => 0,
 631         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 632         optional => 1,
 633     },
 634     tag => {
 635         type => 'integer',
 636         minimum => 1, maximum => 4094,
 637         description => 'VLAN tag to apply to packets on this interface.',
 638         optional => 1,
 639     },
 640     trunks => {
 641         type => 'string',
 642         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 643         description => 'VLAN trunks to pass through this interface.',
 644         format_description => 'vlanid[;vlanid...]',
 645         optional => 1,
 646     },
 647     firewall => {
 648         type => 'boolean',
 649         description => 'Whether this interface should be protected by the firewall.',
 650         optional => 1,
 651     },
 652     link_down => {
 653         type => 'boolean',
 654         description => 'Whether this interface should be disconnected (like pulling the plug).',
 655         optional => 1,
 656     },
 657 };
 658
 659 my $netdesc = {
 660     optional => 1,
 661     type => 'string', format => $net_fmt,
 662     description => "Specify network devices.",
 663 };
 664
 665 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 666
 667 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 668     $confdesc->{"net$i"} = $netdesc;
 669 }
 670
 671 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 672 sub verify_volume_id_or_qm_path {
 673     my ($volid, $noerr) = @_;
 674
 675     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 676         return $volid;
 677     }
 678
 679     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 680     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 681     if ($@) {
 682         return undef if $noerr;
 683         die $@;
 684     }
 685     return $volid;
 686 }
 687
 688 my $drivename_hash;
 689
 690 my %drivedesc_base = (
 691     volume => { alias => 'file' },
 692     file => {
 693         type => 'string',
 694         format => 'pve-volume-id-or-qm-path',
 695         default_key => 1,
 696         format_description => 'volume',
 697         description => "The drive's backing volume.",
 698     },
 699     media => {
 700         type => 'string',
 701         enum => [qw(cdrom disk)],
 702         description => "The drive's media type.",
 703         default => 'disk',
 704         optional => 1
 705     },
 706     cyls => {
 707         type => 'integer',
 708         description => "Force the drive's physical geometry to have a specific cylinder count.",
 709         optional => 1
 710     },
 711     heads => {
 712         type => 'integer',
 713         description => "Force the drive's physical geometry to have a specific head count.",
 714         optional => 1
 715     },
 716     secs => {
 717         type => 'integer',
 718         description => "Force the drive's physical geometry to have a specific sector count.",
 719         optional => 1
 720     },
 721     trans => {
 722         type => 'string',
 723         enum => [qw(none lba auto)],
 724         description => "Force disk geometry bios translation mode.",
 725         optional => 1,
 726     },
 727     snapshot => {
 728         type => 'boolean',
 729         description => "Whether the drive should be included when making snapshots.",
 730         optional => 1,
 731     },
 732     cache => {
 733         type => 'string',
 734         enum => [qw(none writethrough writeback unsafe directsync)],
 735         description => "The drive's cache mode",
 736         optional => 1,
 737     },
 738     format => get_standard_option('pve-qm-image-format'),
 739     size => {
 740         type => 'string',
 741         format => 'disk-size',
 742         format_description => 'DiskSize',
 743         description => "Disk size. This is purely informational and has no effect.",
 744         optional => 1,
 745     },
 746     backup => {
 747         type => 'boolean',
 748         description => "Whether the drive should be included when making backups.",
 749         optional => 1,
 750     },
 751     replicate => {
 752         type => 'boolean',
 753         description => 'Will include this drive to an storage replical job.',
 754         optional => 1,
 755         default => 1,
 756     },
 757     rerror => {
 758         type => 'string',
 759         enum => [qw(ignore report stop)],
 760         description => 'Read error action.',
 761         optional => 1,
 762     },
 763     werror => {
 764         type => 'string',
 765         enum => [qw(enospc ignore report stop)],
 766         description => 'Write error action.',
 767         optional => 1,
 768     },
 769     aio => {
 770         type => 'string',
 771         enum => [qw(native threads)],
 772         description => 'AIO type to use.',
 773         optional => 1,
 774     },
 775     discard => {
 776         type => 'string',
 777         enum => [qw(ignore on)],
 778         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 779         optional => 1,
 780     },
 781     detect_zeroes => {
 782         type => 'boolean',
 783         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 784         optional => 1,
 785     },
 786     serial => {
 787         type => 'string',
 788         format => 'urlencoded',
 789         format_description => 'serial',
 790         maxLength => 20*3, # *3 since it's %xx url enoded
 791         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 792         optional => 1,
 793     }
 794 );
 795
 796 my %iothread_fmt = ( iothread => {
 797         type => 'boolean',
 798         description => "Whether to use iothreads for this drive",
 799         optional => 1,
 800 });
 801
 802 my %model_fmt = (
 803     model => {
 804         type => 'string',
 805         format => 'urlencoded',
 806         format_description => 'model',
 807         maxLength => 40*3, # *3 since it's %xx url enoded
 808         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 809         optional => 1,
 810     },
 811 );
 812
 813 my %queues_fmt = (
 814     queues => {
 815         type => 'integer',
 816         description => "Number of queues.",
 817         minimum => 2,
 818         optional => 1
 819     }
 820 );
 821
 822 my %scsiblock_fmt = (
 823     scsiblock => {
 824         type => 'boolean',
 825         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 826         optional => 1,
 827         default => 0,
 828     },
 829 );
 830
 831 my $add_throttle_desc = sub {
 832     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 833     my $d = {
 834         type => $type,
 835         format_description => $unit,
 836         description => "Maximum $what in $longunit.",
 837         optional => 1,
 838     };
 839     $d->{minimum} = $minimum if defined($minimum);
 840     $drivedesc_base{$key} = $d;
 841 };
 842 # throughput: (leaky bucket)
 843 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 844 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 845 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 846 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 847 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 848 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 849 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 850 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 851 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 852
 853 # pools: (pool of IO before throttling starts taking effect)
 854 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 855 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 856 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 857 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 858 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 859 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 860
 861 # burst lengths
 862 $add_throttle_desc->('bps_max_length',  'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 863 $add_throttle_desc->('bps_rd_length',   'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 864 $add_throttle_desc->('bps_wr_length',   'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 865 $add_throttle_desc->('iops_max_length', 'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 866 $add_throttle_desc->('iops_rd_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 867 $add_throttle_desc->('iops_wr_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 868
 869 my $ide_fmt = {
 870     %drivedesc_base,
 871     %model_fmt,
 872 };
 873 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 874
 875 my $idedesc = {
 876     optional => 1,
 877     type => 'string', format => $ide_fmt,
 878     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 879 };
 880 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 881
 882 my $scsi_fmt = {
 883     %drivedesc_base,
 884     %iothread_fmt,
 885     %queues_fmt,
 886     %scsiblock_fmt,
 887 };
 888 my $scsidesc = {
 889     optional => 1,
 890     type => 'string', format => $scsi_fmt,
 891     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 892 };
 893 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 894
 895 my $sata_fmt = {
 896     %drivedesc_base,
 897 };
 898 my $satadesc = {
 899     optional => 1,
 900     type => 'string', format => $sata_fmt,
 901     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 902 };
 903 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 904
 905 my $virtio_fmt = {
 906     %drivedesc_base,
 907     %iothread_fmt,
 908 };
 909 my $virtiodesc = {
 910     optional => 1,
 911     type => 'string', format => $virtio_fmt,
 912     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 913 };
 914 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 915
 916 my $alldrive_fmt = {
 917     %drivedesc_base,
 918     %iothread_fmt,
 919     %model_fmt,
 920     %queues_fmt,
 921     %scsiblock_fmt,
 922 };
 923
 924 my $efidisk_fmt = {
 925     volume => { alias => 'file' },
 926     file => {
 927         type => 'string',
 928         format => 'pve-volume-id-or-qm-path',
 929         default_key => 1,
 930         format_description => 'volume',
 931         description => "The drive's backing volume.",
 932     },
 933     format => get_standard_option('pve-qm-image-format'),
 934     size => {
 935         type => 'string',
 936         format => 'disk-size',
 937         format_description => 'DiskSize',
 938         description => "Disk size. This is purely informational and has no effect.",
 939         optional => 1,
 940     },
 941 };
 942
 943 my $efidisk_desc = {
 944     optional => 1,
 945     type => 'string', format => $efidisk_fmt,
 946     description => "Configure a Disk for storing EFI vars",
 947 };
 948
 949 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 950
 951 my $usb_fmt = {
 952     host => {
 953         default_key => 1,
 954         type => 'string', format => 'pve-qm-usb-device',
 955         format_description => 'HOSTUSBDEVICE|spice',
 956         description => <<EODESCR,
 957 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 958
 959  'bus-port(.port)*' (decimal numbers) or
 960  'vendor_id:product_id' (hexadeciaml numbers) or
 961  'spice'
 962
 963 You can use the 'lsusb -t' command to list existing usb devices.
 964
 965 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 966
 967 The value 'spice' can be used to add a usb redirection devices for spice.
 968 EODESCR
 969     },
 970     usb3 => {
 971         optional => 1,
 972         type => 'boolean',
 973         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 974         default => 0,
 975     },
 976 };
 977
 978 my $usbdesc = {
 979     optional => 1,
 980     type => 'string', format => $usb_fmt,
 981     description => "Configure an USB device (n is 0 to 4).",
 982 };
 983 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 984
 985 # NOTE: the match-groups of this regex are used in parse_hostpci
 986 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 987 my $hostpci_fmt = {
 988     host => {
 989         default_key => 1,
 990         type => 'string',
 991         pattern => qr/$PCIRE(;$PCIRE)*/,
 992         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 993         description => <<EODESCR,
 994 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 995 of PCI virtual functions of the host. HOSTPCIID syntax is:
 996
 997 'bus:dev.func' (hexadecimal numbers)
 998
 999 You can us the 'lspci' command to list existing PCI devices.
1000 EODESCR
1001     },
1002     rombar => {
1003         type => 'boolean',
1004         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1005         optional => 1,
1006         default => 1,
1007     },
1008     romfile => {
1009         type => 'string',
1010         pattern => '[^,;]+',
1011         format_description => 'string',
1012         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1013         optional => 1,
1014     },
1015     pcie => {
1016         type => 'boolean',
1017         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1018         optional => 1,
1019         default => 0,
1020     },
1021     'x-vga' => {
1022         type => 'boolean',
1023         description =>  "Enable vfio-vga device support.",
1024         optional => 1,
1025         default => 0,
1026     },
1027 };
1028 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1029
1030 my $hostpcidesc = {
1031         optional => 1,
1032         type => 'string', format => 'pve-qm-hostpci',
1033         description => "Map host PCI devices into guest.",
1034         verbose_description =>  <<EODESCR,
1035 Map host PCI devices into guest.
1036
1037 NOTE: This option allows direct access to host hardware. So it is no longer
1038 possible to migrate such machines - use with special care.
1039
1040 CAUTION: Experimental! User reported problems with this option.
1041 EODESCR
1042 };
1043 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1044
1045 my $serialdesc = {
1046         optional => 1,
1047         type => 'string',
1048         pattern => '(/dev/.+|socket)',
1049         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1050         verbose_description =>  <<EODESCR,
1051 Create a serial device inside the VM (n is 0 to 3), and pass through a
1052 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1053 host side (use 'qm terminal' to open a terminal connection).
1054
1055 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1056
1057 CAUTION: Experimental! User reported problems with this option.
1058 EODESCR
1059 };
1060
1061 my $paralleldesc= {
1062         optional => 1,
1063         type => 'string',
1064         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1065         description =>  "Map host parallel devices (n is 0 to 2).",
1066         verbose_description =>  <<EODESCR,
1067 Map host parallel devices (n is 0 to 2).
1068
1069 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1070
1071 CAUTION: Experimental! User reported problems with this option.
1072 EODESCR
1073 };
1074
1075 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1076     $confdesc->{"parallel$i"} = $paralleldesc;
1077 }
1078
1079 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1080     $confdesc->{"serial$i"} = $serialdesc;
1081 }
1082
1083 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1084     $confdesc->{"hostpci$i"} = $hostpcidesc;
1085 }
1086
1087 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1088     $drivename_hash->{"ide$i"} = 1;
1089     $confdesc->{"ide$i"} = $idedesc;
1090 }
1091
1092 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1093     $drivename_hash->{"sata$i"} = 1;
1094     $confdesc->{"sata$i"} = $satadesc;
1095 }
1096
1097 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1098     $drivename_hash->{"scsi$i"} = 1;
1099     $confdesc->{"scsi$i"} = $scsidesc ;
1100 }
1101
1102 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1103     $drivename_hash->{"virtio$i"} = 1;
1104     $confdesc->{"virtio$i"} = $virtiodesc;
1105 }
1106
1107 $drivename_hash->{efidisk0} = 1;
1108 $confdesc->{efidisk0} = $efidisk_desc;
1109
1110 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1111     $confdesc->{"usb$i"} = $usbdesc;
1112 }
1113
1114 my $unuseddesc = {
1115     optional => 1,
1116     type => 'string', format => 'pve-volume-id',
1117     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1118 };
1119
1120 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1121     $confdesc->{"unused$i"} = $unuseddesc;
1122 }
1123
1124 my $kvm_api_version = 0;
1125
1126 sub kvm_version {
1127
1128     return $kvm_api_version if $kvm_api_version;
1129
1130     my $fh = IO::File->new("</dev/kvm") ||
1131         return 0;
1132
1133     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1134         $kvm_api_version = $v;
1135     }
1136
1137     $fh->close();
1138
1139     return  $kvm_api_version;
1140 }
1141
1142 my $kvm_user_version;
1143
1144 sub kvm_user_version {
1145
1146     return $kvm_user_version if $kvm_user_version;
1147
1148     $kvm_user_version = 'unknown';
1149
1150     my $code = sub {
1151         my $line = shift;
1152         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1153             $kvm_user_version = $2;
1154         }
1155     };
1156
1157     eval { run_command("kvm -version", outfunc => $code); };
1158     warn $@ if $@;
1159
1160     return $kvm_user_version;
1161
1162 }
1163
1164 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1165
1166 sub valid_drive_names {
1167     # order is important - used to autoselect boot disk
1168     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1169             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1170             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1171             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1172             'efidisk0');
1173 }
1174
1175 sub is_valid_drivename {
1176     my $dev = shift;
1177
1178     return defined($drivename_hash->{$dev});
1179 }
1180
1181 sub option_exists {
1182     my $key = shift;
1183     return defined($confdesc->{$key});
1184 }
1185
1186 sub nic_models {
1187     return $nic_model_list;
1188 }
1189
1190 sub os_list_description {
1191
1192     return {
1193         other => 'Other',
1194         wxp => 'Windows XP',
1195         w2k => 'Windows 2000',
1196         w2k3 =>, 'Windows 2003',
1197         w2k8 => 'Windows 2008',
1198         wvista => 'Windows Vista',
1199         win7 => 'Windows 7',
1200         win8 => 'Windows 8/2012',
1201         win10 => 'Windows 10/2016',
1202         l24 => 'Linux 2.4',
1203         l26 => 'Linux 2.6',
1204     };
1205 }
1206
1207 my $cdrom_path;
1208
1209 sub get_cdrom_path {
1210
1211     return  $cdrom_path if $cdrom_path;
1212
1213     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1214     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1215     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1216 }
1217
1218 sub get_iso_path {
1219     my ($storecfg, $vmid, $cdrom) = @_;
1220
1221     if ($cdrom eq 'cdrom') {
1222         return get_cdrom_path();
1223     } elsif ($cdrom eq 'none') {
1224         return '';
1225     } elsif ($cdrom =~ m|^/|) {
1226         return $cdrom;
1227     } else {
1228         return PVE::Storage::path($storecfg, $cdrom);
1229     }
1230 }
1231
1232 # try to convert old style file names to volume IDs
1233 sub filename_to_volume_id {
1234     my ($vmid, $file, $media) = @_;
1235
1236     if (!($file eq 'none' || $file eq 'cdrom' ||
1237           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1238
1239         return undef if $file =~ m|/|;
1240
1241         if ($media && $media eq 'cdrom') {
1242             $file = "local:iso/$file";
1243         } else {
1244             $file = "local:$vmid/$file";
1245         }
1246     }
1247
1248     return $file;
1249 }
1250
1251 sub verify_media_type {
1252     my ($opt, $vtype, $media) = @_;
1253
1254     return if !$media;
1255
1256     my $etype;
1257     if ($media eq 'disk') {
1258         $etype = 'images';
1259     } elsif ($media eq 'cdrom') {
1260         $etype = 'iso';
1261     } else {
1262         die "internal error";
1263     }
1264
1265     return if ($vtype eq $etype);
1266
1267     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1268 }
1269
1270 sub cleanup_drive_path {
1271     my ($opt, $storecfg, $drive) = @_;
1272
1273     # try to convert filesystem paths to volume IDs
1274
1275     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1276         ($drive->{file} !~ m|^/dev/.+|) &&
1277         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1278         ($drive->{file} !~ m/^\d+$/)) {
1279         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1280         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1281         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1282         verify_media_type($opt, $vtype, $drive->{media});
1283         $drive->{file} = $volid;
1284     }
1285
1286     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1287 }
1288
1289 sub parse_hotplug_features {
1290     my ($data) = @_;
1291
1292     my $res = {};
1293
1294     return $res if $data eq '0';
1295
1296     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1297
1298     foreach my $feature (PVE::Tools::split_list($data)) {
1299         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1300             $res->{$1} = 1;
1301         } else {
1302             die "invalid hotplug feature '$feature'\n";
1303         }
1304     }
1305     return $res;
1306 }
1307
1308 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1309 sub pve_verify_hotplug_features {
1310     my ($value, $noerr) = @_;
1311
1312     return $value if parse_hotplug_features($value);
1313
1314     return undef if $noerr;
1315
1316     die "unable to parse hotplug option\n";
1317 }
1318
1319 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1320 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1321 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1322 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1323 #        [,iothread=on][,serial=serial][,model=model]
1324
1325 sub parse_drive {
1326     my ($key, $data) = @_;
1327
1328     my ($interface, $index);
1329
1330     if ($key =~ m/^([^\d]+)(\d+)$/) {
1331         $interface = $1;
1332         $index = $2;
1333     } else {
1334         return undef;
1335     }
1336
1337     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1338                                      : $confdesc->{$key}->{format};
1339     if (!$desc) {
1340         warn "invalid drive key: $key\n";
1341         return undef;
1342     }
1343     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1344     return undef if !$res;
1345     $res->{interface} = $interface;
1346     $res->{index} = $index;
1347
1348     my $error = 0;
1349     foreach my $opt (qw(bps bps_rd bps_wr)) {
1350         if (my $bps = defined(delete $res->{$opt})) {
1351             if (defined($res->{"m$opt"})) {
1352                 warn "both $opt and m$opt specified\n";
1353                 ++$error;
1354                 next;
1355             }
1356             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1357         }
1358     }
1359
1360     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1361     for my $requirement (
1362         [bps_max_length => 'mbps_max'],
1363         [bps_rd_max_length => 'mbps_rd_max'],
1364         [bps_wr_max_length => 'mbps_wr_max'],
1365         [iops_max_length => 'iops_max'],
1366         [iops_rd_max_length => 'iops_rd_max'],
1367         [iops_wr_max_length => 'iops_wr_max']) {
1368         my ($option, $requires) = @$requirement;
1369         if ($res->{$option} && !$res->{$requires}) {
1370             warn "$option requires $requires\n";
1371             ++$error;
1372         }
1373     }
1374
1375     return undef if $error;
1376
1377     return undef if $res->{mbps_rd} && $res->{mbps};
1378     return undef if $res->{mbps_wr} && $res->{mbps};
1379     return undef if $res->{iops_rd} && $res->{iops};
1380     return undef if $res->{iops_wr} && $res->{iops};
1381
1382     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1383         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1384         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1385         return undef if $res->{interface} eq 'virtio';
1386     }
1387
1388     if (my $size = $res->{size}) {
1389         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1390     }
1391
1392     return $res;
1393 }
1394
1395 sub print_drive {
1396     my ($vmid, $drive) = @_;
1397     my $data = { %$drive };
1398     delete $data->{$_} for qw(index interface);
1399     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1400 }
1401
1402 sub scsi_inquiry {
1403     my($fh, $noerr) = @_;
1404
1405     my $SG_IO = 0x2285;
1406     my $SG_GET_VERSION_NUM = 0x2282;
1407
1408     my $versionbuf = "\x00" x 8;
1409     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1410     if (!$ret) {
1411         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1412         return undef;
1413     }
1414     my $version = unpack("I", $versionbuf);
1415     if ($version < 30000) {
1416         die "scsi generic interface too old\n"  if !$noerr;
1417         return undef;
1418     }
1419
1420     my $buf = "\x00" x 36;
1421     my $sensebuf = "\x00" x 8;
1422     my $cmd = pack("C x3 C x1", 0x12, 36);
1423
1424     # see /usr/include/scsi/sg.h
1425     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1426
1427     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1428                       length($sensebuf), 0, length($buf), $buf,
1429                       $cmd, $sensebuf, 6000);
1430
1431     $ret = ioctl($fh, $SG_IO, $packet);
1432     if (!$ret) {
1433         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1434         return undef;
1435     }
1436
1437     my @res = unpack($sg_io_hdr_t, $packet);
1438     if ($res[17] || $res[18]) {
1439         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1440         return undef;
1441     }
1442
1443     my $res = {};
1444     (my $byte0, my $byte1, $res->{vendor},
1445      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1446
1447     $res->{removable} = $byte1 & 128 ? 1 : 0;
1448     $res->{type} = $byte0 & 31;
1449
1450     return $res;
1451 }
1452
1453 sub path_is_scsi {
1454     my ($path) = @_;
1455
1456     my $fh = IO::File->new("+<$path") || return undef;
1457     my $res = scsi_inquiry($fh, 1);
1458     close($fh);
1459
1460     return $res;
1461 }
1462
1463 sub machine_type_is_q35 {
1464     my ($conf) = @_;
1465
1466     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1467 }
1468
1469 sub print_tabletdevice_full {
1470     my ($conf) = @_;
1471
1472     my $q35 = machine_type_is_q35($conf);
1473
1474     # we use uhci for old VMs because tablet driver was buggy in older qemu
1475     my $usbbus = $q35 ? "ehci" : "uhci";
1476
1477     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1478 }
1479
1480 sub print_drivedevice_full {
1481     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1482
1483     my $device = '';
1484     my $maxdev = 0;
1485
1486     if ($drive->{interface} eq 'virtio') {
1487         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1488         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1489         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1490     } elsif ($drive->{interface} eq 'scsi') {
1491
1492         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1493         my $unit = $drive->{index} % $maxdev;
1494         my $devicetype = 'hd';
1495         my $path = '';
1496         if (drive_is_cdrom($drive)) {
1497             $devicetype = 'cd';
1498         } else {
1499             if ($drive->{file} =~ m|^/|) {
1500                 $path = $drive->{file};
1501                 if (my $info = path_is_scsi($path)) {
1502                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1503                         $devicetype = 'block';
1504                     } elsif ($info->{type} == 1) { # tape
1505                         $devicetype = 'generic';
1506                     }
1507                 }
1508             } else {
1509                  $path = PVE::Storage::path($storecfg, $drive->{file});
1510             }
1511
1512             if($path =~ m/^iscsi\:\/\//){
1513                 $devicetype = 'generic';
1514             }
1515         }
1516
1517         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1518             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1519         } else {
1520             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1521         }
1522
1523     } elsif ($drive->{interface} eq 'ide'){
1524         $maxdev = 2;
1525         my $controller = int($drive->{index} / $maxdev);
1526         my $unit = $drive->{index} % $maxdev;
1527         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1528
1529         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1530         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1531             $model = URI::Escape::uri_unescape($model);
1532             $device .= ",model=$model";
1533         }
1534     } elsif ($drive->{interface} eq 'sata'){
1535         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1536         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1537         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1538     } elsif ($drive->{interface} eq 'usb') {
1539         die "implement me";
1540         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1541     } else {
1542         die "unsupported interface type";
1543     }
1544
1545     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1546
1547     return $device;
1548 }
1549
1550 sub get_initiator_name {
1551     my $initiator;
1552
1553     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1554     while (defined(my $line = <$fh>)) {
1555         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1556         $initiator = $1;
1557         last;
1558     }
1559     $fh->close();
1560
1561     return $initiator;
1562 }
1563
1564 sub print_drive_full {
1565     my ($storecfg, $vmid, $drive) = @_;
1566
1567     my $path;
1568     my $volid = $drive->{file};
1569     my $format;
1570
1571     if (drive_is_cdrom($drive)) {
1572         $path = get_iso_path($storecfg, $vmid, $volid);
1573     } else {
1574         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1575         if ($storeid) {
1576             $path = PVE::Storage::path($storecfg, $volid);
1577             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1578             $format = qemu_img_format($scfg, $volname);
1579         } else {
1580             $path = $volid;
1581             $format = "raw";
1582         }
1583    }
1584
1585     my $opts = '';
1586     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1587     foreach my $o (@qemu_drive_options) {
1588         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1589     }
1590     if (my $serial = $drive->{serial}) {
1591         $serial = URI::Escape::uri_unescape($serial);
1592         $opts .= ",serial=$serial";
1593     }
1594
1595     $opts .= ",format=$format" if $format && !$drive->{format};
1596
1597     foreach my $o (qw(bps bps_rd bps_wr)) {
1598         my $v = $drive->{"m$o"};
1599         $opts .= ",$o=" . int($v*1024*1024) if $v;
1600     }
1601
1602     my $cache_direct = 0;
1603
1604     if (my $cache = $drive->{cache}) {
1605         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1606     } elsif (!drive_is_cdrom($drive)) {
1607         $opts .= ",cache=none";
1608         $cache_direct = 1;
1609     }
1610
1611     # aio native works only with O_DIRECT
1612     if (!$drive->{aio}) {
1613         if($cache_direct) {
1614             $opts .= ",aio=native";
1615         } else {
1616             $opts .= ",aio=threads";
1617         }
1618     }
1619
1620     if (!drive_is_cdrom($drive)) {
1621         my $detectzeroes;
1622         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1623             $detectzeroes = 'off';
1624         } elsif ($drive->{discard}) {
1625             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1626         } else {
1627             # This used to be our default with discard not being specified:
1628             $detectzeroes = 'on';
1629         }
1630         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1631     }
1632
1633     my $pathinfo = $path ? "file=$path," : '';
1634
1635     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1636 }
1637
1638 sub print_netdevice_full {
1639     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1640
1641     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1642
1643     my $device = $net->{model};
1644     if ($net->{model} eq 'virtio') {
1645          $device = 'virtio-net-pci';
1646      };
1647
1648     my $pciaddr = print_pci_addr("$netid", $bridges);
1649     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1650     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1651         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1652         my $vectors = $net->{queues} * 2 + 2;
1653         $tmpstr .= ",vectors=$vectors,mq=on";
1654     }
1655     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1656
1657     if ($use_old_bios_files) {
1658         my $romfile;
1659         if ($device eq 'virtio-net-pci') {
1660             $romfile = 'pxe-virtio.rom';
1661         } elsif ($device eq 'e1000') {
1662             $romfile = 'pxe-e1000.rom';
1663         } elsif ($device eq 'ne2k') {
1664             $romfile = 'pxe-ne2k_pci.rom';
1665         } elsif ($device eq 'pcnet') {
1666             $romfile = 'pxe-pcnet.rom';
1667         } elsif ($device eq 'rtl8139') {
1668             $romfile = 'pxe-rtl8139.rom';
1669         }
1670         $tmpstr .= ",romfile=$romfile" if $romfile;
1671     }
1672
1673     return $tmpstr;
1674 }
1675
1676 sub print_netdev_full {
1677     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1678
1679     my $i = '';
1680     if ($netid =~ m/^net(\d+)$/) {
1681         $i = int($1);
1682     }
1683
1684     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1685
1686     my $ifname = "tap${vmid}i$i";
1687
1688     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1689     die "interface name '$ifname' is too long (max 15 character)\n"
1690         if length($ifname) >= 16;
1691
1692     my $vhostparam = '';
1693     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1694
1695     my $vmname = $conf->{name} || "vm$vmid";
1696
1697     my $netdev = "";
1698     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1699
1700     if ($net->{bridge}) {
1701         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1702     } else {
1703         $netdev = "type=user,id=$netid,hostname=$vmname";
1704     }
1705
1706     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1707
1708     return $netdev;
1709 }
1710
1711
1712 sub print_cpu_device {
1713     my ($conf, $id) = @_;
1714
1715     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
1716     my $cpu = $nokvm ? "qemu64" : "kvm64";
1717     if (my $cputype = $conf->{cpu}) {
1718         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1719             or die "Cannot parse cpu description: $cputype\n";
1720         $cpu = $cpuconf->{cputype};
1721     }
1722
1723     my $cores = $conf->{cores} || 1;
1724
1725     my $current_core = ($id - 1) % $cores;
1726     my $current_socket = int(($id - 1 - $current_core)/$cores);
1727
1728     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1729 }
1730
1731 sub drive_is_cdrom {
1732     my ($drive) = @_;
1733
1734     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1735
1736 }
1737
1738 sub parse_number_sets {
1739     my ($set) = @_;
1740     my $res = [];
1741     foreach my $part (split(/;/, $set)) {
1742         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1743             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1744             push @$res, [ $1, $2 ];
1745         } else {
1746             die "invalid range: $part\n";
1747         }
1748     }
1749     return $res;
1750 }
1751
1752 sub parse_numa {
1753     my ($data) = @_;
1754
1755     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1756     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1757     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1758     return $res;
1759 }
1760
1761 sub parse_hostpci {
1762     my ($value) = @_;
1763
1764     return undef if !$value;
1765
1766     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1767
1768     my @idlist = split(/;/, $res->{host});
1769     delete $res->{host};
1770     foreach my $id (@idlist) {
1771         if ($id =~ /^$PCIRE$/) {
1772             if (defined($2)) {
1773                 push @{$res->{pciid}}, { id => $1, function => $2 };
1774             } else {
1775                 my $pcidevices = lspci($1);
1776                 $res->{pciid} = $pcidevices->{$1};
1777             }
1778         } else {
1779             # should have been caught by parse_property_string already
1780             die "failed to parse PCI id: $id\n";
1781         }
1782     }
1783     return $res;
1784 }
1785
1786 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1787 sub parse_net {
1788     my ($data) = @_;
1789
1790     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1791     if ($@) {
1792         warn $@;
1793         return undef;
1794     }
1795     if (!defined($res->{macaddr})) {
1796         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1797         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1798     }
1799     return $res;
1800 }
1801
1802 sub print_net {
1803     my $net = shift;
1804
1805     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1806 }
1807
1808 sub add_random_macs {
1809     my ($settings) = @_;
1810
1811     foreach my $opt (keys %$settings) {
1812         next if $opt !~ m/^net(\d+)$/;
1813         my $net = parse_net($settings->{$opt});
1814         next if !$net;
1815         $settings->{$opt} = print_net($net);
1816     }
1817 }
1818
1819 sub vm_is_volid_owner {
1820     my ($storecfg, $vmid, $volid) = @_;
1821
1822     if ($volid !~  m|^/|) {
1823         my ($path, $owner);
1824         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1825         if ($owner && ($owner == $vmid)) {
1826             return 1;
1827         }
1828     }
1829
1830     return undef;
1831 }
1832
1833 sub split_flagged_list {
1834     my $text = shift || '';
1835     $text =~ s/[,;]/ /g;
1836     $text =~ s/^\s+//;
1837     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1838 }
1839
1840 sub join_flagged_list {
1841     my ($how, $lst) = @_;
1842     join $how, map { $lst->{$_} . $_ } keys %$lst;
1843 }
1844
1845 sub vmconfig_delete_pending_option {
1846     my ($conf, $key, $force) = @_;
1847
1848     delete $conf->{pending}->{$key};
1849     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1850     $pending_delete_hash->{$key} = $force ? '!' : '';
1851     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1852 }
1853
1854 sub vmconfig_undelete_pending_option {
1855     my ($conf, $key) = @_;
1856
1857     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1858     delete $pending_delete_hash->{$key};
1859
1860     if (%$pending_delete_hash) {
1861         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1862     } else {
1863         delete $conf->{pending}->{delete};
1864     }
1865 }
1866
1867 sub vmconfig_register_unused_drive {
1868     my ($storecfg, $vmid, $conf, $drive) = @_;
1869
1870     if (!drive_is_cdrom($drive)) {
1871         my $volid = $drive->{file};
1872         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1873             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1874         }
1875     }
1876 }
1877
1878 sub vmconfig_cleanup_pending {
1879     my ($conf) = @_;
1880
1881     # remove pending changes when nothing changed
1882     my $changes;
1883     foreach my $opt (keys %{$conf->{pending}}) {
1884         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1885             $changes = 1;
1886             delete $conf->{pending}->{$opt};
1887         }
1888     }
1889
1890     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1891     my $pending_delete_hash = {};
1892     while (my ($opt, $force) = each %$current_delete_hash) {
1893         if (defined($conf->{$opt})) {
1894             $pending_delete_hash->{$opt} = $force;
1895         } else {
1896             $changes = 1;
1897         }
1898     }
1899
1900     if (%$pending_delete_hash) {
1901         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1902     } else {
1903         delete $conf->{pending}->{delete};
1904     }
1905
1906     return $changes;
1907 }
1908
1909 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1910 my $smbios1_fmt = {
1911     uuid => {
1912         type => 'string',
1913         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1914         format_description => 'UUID',
1915         description => "Set SMBIOS1 UUID.",
1916         optional => 1,
1917     },
1918     version => {
1919         type => 'string',
1920         pattern => '\S+',
1921         format_description => 'string',
1922         description => "Set SMBIOS1 version.",
1923         optional => 1,
1924     },
1925     serial => {
1926         type => 'string',
1927         pattern => '\S+',
1928         format_description => 'string',
1929         description => "Set SMBIOS1 serial number.",
1930         optional => 1,
1931     },
1932     manufacturer => {
1933         type => 'string',
1934         pattern => '\S+',
1935         format_description => 'string',
1936         description => "Set SMBIOS1 manufacturer.",
1937         optional => 1,
1938     },
1939     product => {
1940         type => 'string',
1941         pattern => '\S+',
1942         format_description => 'string',
1943         description => "Set SMBIOS1 product ID.",
1944         optional => 1,
1945     },
1946     sku => {
1947         type => 'string',
1948         pattern => '\S+',
1949         format_description => 'string',
1950         description => "Set SMBIOS1 SKU string.",
1951         optional => 1,
1952     },
1953     family => {
1954         type => 'string',
1955         pattern => '\S+',
1956         format_description => 'string',
1957         description => "Set SMBIOS1 family string.",
1958         optional => 1,
1959     },
1960 };
1961
1962 sub parse_smbios1 {
1963     my ($data) = @_;
1964
1965     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1966     warn $@ if $@;
1967     return $res;
1968 }
1969
1970 sub print_smbios1 {
1971     my ($smbios1) = @_;
1972     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1973 }
1974
1975 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1976
1977 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1978 sub verify_bootdisk {
1979     my ($value, $noerr) = @_;
1980
1981     return $value if is_valid_drivename($value);
1982
1983     return undef if $noerr;
1984
1985     die "invalid boot disk '$value'\n";
1986 }
1987
1988 sub parse_watchdog {
1989     my ($value) = @_;
1990
1991     return undef if !$value;
1992
1993     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1994     warn $@ if $@;
1995     return $res;
1996 }
1997
1998 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1999 sub verify_usb_device {
2000     my ($value, $noerr) = @_;
2001
2002     return $value if parse_usb_device($value);
2003
2004     return undef if $noerr;
2005
2006     die "unable to parse usb device\n";
2007 }
2008
2009 # add JSON properties for create and set function
2010 sub json_config_properties {
2011     my $prop = shift;
2012
2013     foreach my $opt (keys %$confdesc) {
2014         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2015         $prop->{$opt} = $confdesc->{$opt};
2016     }
2017
2018     return $prop;
2019 }
2020
2021 sub check_type {
2022     my ($key, $value) = @_;
2023
2024     die "unknown setting '$key'\n" if !$confdesc->{$key};
2025
2026     my $type = $confdesc->{$key}->{type};
2027
2028     if (!defined($value)) {
2029         die "got undefined value\n";
2030     }
2031
2032     if ($value =~ m/[\n\r]/) {
2033         die "property contains a line feed\n";
2034     }
2035
2036     if ($type eq 'boolean') {
2037         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2038         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2039         die "type check ('boolean') failed - got '$value'\n";
2040     } elsif ($type eq 'integer') {
2041         return int($1) if $value =~ m/^(\d+)$/;
2042         die "type check ('integer') failed - got '$value'\n";
2043     } elsif ($type eq 'number') {
2044         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2045         die "type check ('number') failed - got '$value'\n";
2046     } elsif ($type eq 'string') {
2047         if (my $fmt = $confdesc->{$key}->{format}) {
2048             PVE::JSONSchema::check_format($fmt, $value);
2049             return $value;
2050         }
2051         $value =~ s/^\"(.*)\"$/$1/;
2052         return $value;
2053     } else {
2054         die "internal error"
2055     }
2056 }
2057
2058 sub check_iommu_support{
2059     #fixme : need to check IOMMU support
2060     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2061
2062     my $iommu=1;
2063     return $iommu;
2064
2065 }
2066
2067 sub touch_config {
2068     my ($vmid) = @_;
2069
2070     my $conf = PVE::QemuConfig->config_file($vmid);
2071     utime undef, undef, $conf;
2072 }
2073
2074 sub destroy_vm {
2075     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2076
2077     my $conffile = PVE::QemuConfig->config_file($vmid);
2078
2079     my $conf = PVE::QemuConfig->load_config($vmid);
2080
2081     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2082
2083     # only remove disks owned by this VM
2084     foreach_drive($conf, sub {
2085         my ($ds, $drive) = @_;
2086
2087         return if drive_is_cdrom($drive);
2088
2089         my $volid = $drive->{file};
2090
2091         return if !$volid || $volid =~ m|^/|;
2092
2093         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2094         return if !$path || !$owner || ($owner != $vmid);
2095
2096         eval {
2097             PVE::Storage::vdisk_free($storecfg, $volid);
2098         };
2099         warn "Could not remove disk '$volid', check manually: $@" if $@;
2100
2101     });
2102
2103     if ($keep_empty_config) {
2104         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2105     } else {
2106         unlink $conffile;
2107     }
2108
2109     # also remove unused disk
2110     eval {
2111         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2112
2113         eval {
2114             PVE::Storage::foreach_volid($dl, sub {
2115                 my ($volid, $sid, $volname, $d) = @_;
2116                 PVE::Storage::vdisk_free($storecfg, $volid);
2117             });
2118         };
2119         warn $@ if $@;
2120
2121     };
2122     warn $@ if $@;
2123 }
2124
2125 sub parse_vm_config {
2126     my ($filename, $raw) = @_;
2127
2128     return undef if !defined($raw);
2129
2130     my $res = {
2131         digest => Digest::SHA::sha1_hex($raw),
2132         snapshots => {},
2133         pending => {},
2134     };
2135
2136     $filename =~ m|/qemu-server/(\d+)\.conf$|
2137         || die "got strange filename '$filename'";
2138
2139     my $vmid = $1;
2140
2141     my $conf = $res;
2142     my $descr;
2143     my $section = '';
2144
2145     my @lines = split(/\n/, $raw);
2146     foreach my $line (@lines) {
2147         next if $line =~ m/^\s*$/;
2148
2149         if ($line =~ m/^\[PENDING\]\s*$/i) {
2150             $section = 'pending';
2151             if (defined($descr)) {
2152                 $descr =~ s/\s+$//;
2153                 $conf->{description} = $descr;
2154             }
2155             $descr = undef;
2156             $conf = $res->{$section} = {};
2157             next;
2158
2159         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2160             $section = $1;
2161             if (defined($descr)) {
2162                 $descr =~ s/\s+$//;
2163                 $conf->{description} = $descr;
2164             }
2165             $descr = undef;
2166             $conf = $res->{snapshots}->{$section} = {};
2167             next;
2168         }
2169
2170         if ($line =~ m/^\#(.*)\s*$/) {
2171             $descr = '' if !defined($descr);
2172             $descr .= PVE::Tools::decode_text($1) . "\n";
2173             next;
2174         }
2175
2176         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2177             $descr = '' if !defined($descr);
2178             $descr .= PVE::Tools::decode_text($2);
2179         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2180             $conf->{snapstate} = $1;
2181         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2182             my $key = $1;
2183             my $value = $2;
2184             $conf->{$key} = $value;
2185         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2186             my $value = $1;
2187             if ($section eq 'pending') {
2188                 $conf->{delete} = $value; # we parse this later
2189             } else {
2190                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2191             }
2192         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2193             my $key = $1;
2194             my $value = $2;
2195             eval { $value = check_type($key, $value); };
2196             if ($@) {
2197                 warn "vm $vmid - unable to parse value of '$key' - $@";
2198             } else {
2199                 $key = 'ide2' if $key eq 'cdrom';
2200                 my $fmt = $confdesc->{$key}->{format};
2201                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2202                     my $v = parse_drive($key, $value);
2203                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2204                         $v->{file} = $volid;
2205                         $value = print_drive($vmid, $v);
2206                     } else {
2207                         warn "vm $vmid - unable to parse value of '$key'\n";
2208                         next;
2209                     }
2210                 }
2211
2212                 $conf->{$key} = $value;
2213             }
2214         }
2215     }
2216
2217     if (defined($descr)) {
2218         $descr =~ s/\s+$//;
2219         $conf->{description} = $descr;
2220     }
2221     delete $res->{snapstate}; # just to be sure
2222
2223     return $res;
2224 }
2225
2226 sub write_vm_config {
2227     my ($filename, $conf) = @_;
2228
2229     delete $conf->{snapstate}; # just to be sure
2230
2231     if ($conf->{cdrom}) {
2232         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2233         $conf->{ide2} = $conf->{cdrom};
2234         delete $conf->{cdrom};
2235     }
2236
2237     # we do not use 'smp' any longer
2238     if ($conf->{sockets}) {
2239         delete $conf->{smp};
2240     } elsif ($conf->{smp}) {
2241         $conf->{sockets} = $conf->{smp};
2242         delete $conf->{cores};
2243         delete $conf->{smp};
2244     }
2245
2246     my $used_volids = {};
2247
2248     my $cleanup_config = sub {
2249         my ($cref, $pending, $snapname) = @_;
2250
2251         foreach my $key (keys %$cref) {
2252             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2253                 $key eq 'snapstate' || $key eq 'pending';
2254             my $value = $cref->{$key};
2255             if ($key eq 'delete') {
2256                 die "propertry 'delete' is only allowed in [PENDING]\n"
2257                     if !$pending;
2258                 # fixme: check syntax?
2259                 next;
2260             }
2261             eval { $value = check_type($key, $value); };
2262             die "unable to parse value of '$key' - $@" if $@;
2263
2264             $cref->{$key} = $value;
2265
2266             if (!$snapname && is_valid_drivename($key)) {
2267                 my $drive = parse_drive($key, $value);
2268                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2269             }
2270         }
2271     };
2272
2273     &$cleanup_config($conf);
2274
2275     &$cleanup_config($conf->{pending}, 1);
2276
2277     foreach my $snapname (keys %{$conf->{snapshots}}) {
2278         die "internal error" if $snapname eq 'pending';
2279         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2280     }
2281
2282     # remove 'unusedX' settings if we re-add a volume
2283     foreach my $key (keys %$conf) {
2284         my $value = $conf->{$key};
2285         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2286             delete $conf->{$key};
2287         }
2288     }
2289
2290     my $generate_raw_config = sub {
2291         my ($conf, $pending) = @_;
2292
2293         my $raw = '';
2294
2295         # add description as comment to top of file
2296         if (defined(my $descr = $conf->{description})) {
2297             if ($descr) {
2298                 foreach my $cl (split(/\n/, $descr)) {
2299                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2300                 }
2301             } else {
2302                 $raw .= "#\n" if $pending;
2303             }
2304         }
2305
2306         foreach my $key (sort keys %$conf) {
2307             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2308             $raw .= "$key: $conf->{$key}\n";
2309         }
2310         return $raw;
2311     };
2312
2313     my $raw = &$generate_raw_config($conf);
2314
2315     if (scalar(keys %{$conf->{pending}})){
2316         $raw .= "\n[PENDING]\n";
2317         $raw .= &$generate_raw_config($conf->{pending}, 1);
2318     }
2319
2320     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2321         $raw .= "\n[$snapname]\n";
2322         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2323     }
2324
2325     return $raw;
2326 }
2327
2328 sub load_defaults {
2329
2330     my $res = {};
2331
2332     # we use static defaults from our JSON schema configuration
2333     foreach my $key (keys %$confdesc) {
2334         if (defined(my $default = $confdesc->{$key}->{default})) {
2335             $res->{$key} = $default;
2336         }
2337     }
2338
2339     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2340     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2341
2342     return $res;
2343 }
2344
2345 sub config_list {
2346     my $vmlist = PVE::Cluster::get_vmlist();
2347     my $res = {};
2348     return $res if !$vmlist || !$vmlist->{ids};
2349     my $ids = $vmlist->{ids};
2350
2351     foreach my $vmid (keys %$ids) {
2352         my $d = $ids->{$vmid};
2353         next if !$d->{node} || $d->{node} ne $nodename;
2354         next if !$d->{type} || $d->{type} ne 'qemu';
2355         $res->{$vmid}->{exists} = 1;
2356     }
2357     return $res;
2358 }
2359
2360 # test if VM uses local resources (to prevent migration)
2361 sub check_local_resources {
2362     my ($conf, $noerr) = @_;
2363
2364     my $loc_res = 0;
2365
2366     $loc_res = 1 if $conf->{hostusb}; # old syntax
2367     $loc_res = 1 if $conf->{hostpci}; # old syntax
2368
2369     foreach my $k (keys %$conf) {
2370         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2371         # sockets are safe: they will recreated be on the target side post-migrate
2372         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2373         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2374     }
2375
2376     die "VM uses local resources\n" if $loc_res && !$noerr;
2377
2378     return $loc_res;
2379 }
2380
2381 # check if used storages are available on all nodes (use by migrate)
2382 sub check_storage_availability {
2383     my ($storecfg, $conf, $node) = @_;
2384
2385     foreach_drive($conf, sub {
2386         my ($ds, $drive) = @_;
2387
2388         my $volid = $drive->{file};
2389         return if !$volid;
2390
2391         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2392         return if !$sid;
2393
2394         # check if storage is available on both nodes
2395         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2396         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2397    });
2398 }
2399
2400 # list nodes where all VM images are available (used by has_feature API)
2401 sub shared_nodes {
2402     my ($conf, $storecfg) = @_;
2403
2404     my $nodelist = PVE::Cluster::get_nodelist();
2405     my $nodehash = { map { $_ => 1 } @$nodelist };
2406     my $nodename = PVE::INotify::nodename();
2407
2408     foreach_drive($conf, sub {
2409         my ($ds, $drive) = @_;
2410
2411         my $volid = $drive->{file};
2412         return if !$volid;
2413
2414         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2415         if ($storeid) {
2416             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2417             if ($scfg->{disable}) {
2418                 $nodehash = {};
2419             } elsif (my $avail = $scfg->{nodes}) {
2420                 foreach my $node (keys %$nodehash) {
2421                     delete $nodehash->{$node} if !$avail->{$node};
2422                 }
2423             } elsif (!$scfg->{shared}) {
2424                 foreach my $node (keys %$nodehash) {
2425                     delete $nodehash->{$node} if $node ne $nodename
2426                 }
2427             }
2428         }
2429     });
2430
2431     return $nodehash
2432 }
2433
2434 sub check_cmdline {
2435     my ($pidfile, $pid) = @_;
2436
2437     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2438     if (defined($fh)) {
2439         my $line = <$fh>;
2440         $fh->close;
2441         return undef if !$line;
2442         my @param = split(/\0/, $line);
2443
2444         my $cmd = $param[0];
2445         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2446
2447         for (my $i = 0; $i < scalar (@param); $i++) {
2448             my $p = $param[$i];
2449             next if !$p;
2450             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2451                 my $p = $param[$i+1];
2452                 return 1 if $p && ($p eq $pidfile);
2453                 return undef;
2454             }
2455         }
2456     }
2457     return undef;
2458 }
2459
2460 sub check_running {
2461     my ($vmid, $nocheck, $node) = @_;
2462
2463     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2464
2465     die "unable to find configuration file for VM $vmid - no such machine\n"
2466         if !$nocheck && ! -f $filename;
2467
2468     my $pidfile = pidfile_name($vmid);
2469
2470     if (my $fd = IO::File->new("<$pidfile")) {
2471         my $st = stat($fd);
2472         my $line = <$fd>;
2473         close($fd);
2474
2475         my $mtime = $st->mtime;
2476         if ($mtime > time()) {
2477             warn "file '$filename' modified in future\n";
2478         }
2479
2480         if ($line =~ m/^(\d+)$/) {
2481             my $pid = $1;
2482             if (check_cmdline($pidfile, $pid)) {
2483                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2484                     return $pid;
2485                 }
2486             }
2487         }
2488     }
2489
2490     return undef;
2491 }
2492
2493 sub vzlist {
2494
2495     my $vzlist = config_list();
2496
2497     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2498
2499     while (defined(my $de = $fd->read)) {
2500         next if $de !~ m/^(\d+)\.pid$/;
2501         my $vmid = $1;
2502         next if !defined($vzlist->{$vmid});
2503         if (my $pid = check_running($vmid)) {
2504             $vzlist->{$vmid}->{pid} = $pid;
2505         }
2506     }
2507
2508     return $vzlist;
2509 }
2510
2511 sub disksize {
2512     my ($storecfg, $conf) = @_;
2513
2514     my $bootdisk = $conf->{bootdisk};
2515     return undef if !$bootdisk;
2516     return undef if !is_valid_drivename($bootdisk);
2517
2518     return undef if !$conf->{$bootdisk};
2519
2520     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2521     return undef if !defined($drive);
2522
2523     return undef if drive_is_cdrom($drive);
2524
2525     my $volid = $drive->{file};
2526     return undef if !$volid;
2527
2528     return $drive->{size};
2529 }
2530
2531 my $last_proc_pid_stat;
2532
2533 # get VM status information
2534 # This must be fast and should not block ($full == false)
2535 # We only query KVM using QMP if $full == true (this can be slow)
2536 sub vmstatus {
2537     my ($opt_vmid, $full) = @_;
2538
2539     my $res = {};
2540
2541     my $storecfg = PVE::Storage::config();
2542
2543     my $list = vzlist();
2544     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2545
2546     my $cpucount = $cpuinfo->{cpus} || 1;
2547
2548     foreach my $vmid (keys %$list) {
2549         next if $opt_vmid && ($vmid ne $opt_vmid);
2550
2551         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2552         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2553
2554         my $d = {};
2555         $d->{pid} = $list->{$vmid}->{pid};
2556
2557         # fixme: better status?
2558         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2559
2560         my $size = disksize($storecfg, $conf);
2561         if (defined($size)) {
2562             $d->{disk} = 0; # no info available
2563             $d->{maxdisk} = $size;
2564         } else {
2565             $d->{disk} = 0;
2566             $d->{maxdisk} = 0;
2567         }
2568
2569         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2570         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2571         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2572
2573         $d->{name} = $conf->{name} || "VM $vmid";
2574         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2575
2576         if ($conf->{balloon}) {
2577             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2578             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2579         }
2580
2581         $d->{uptime} = 0;
2582         $d->{cpu} = 0;
2583         $d->{mem} = 0;
2584
2585         $d->{netout} = 0;
2586         $d->{netin} = 0;
2587
2588         $d->{diskread} = 0;
2589         $d->{diskwrite} = 0;
2590
2591         $d->{template} = PVE::QemuConfig->is_template($conf);
2592
2593         $res->{$vmid} = $d;
2594     }
2595
2596     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2597     foreach my $dev (keys %$netdev) {
2598         next if $dev !~ m/^tap([1-9]\d*)i/;
2599         my $vmid = $1;
2600         my $d = $res->{$vmid};
2601         next if !$d;
2602
2603         $d->{netout} += $netdev->{$dev}->{receive};
2604         $d->{netin} += $netdev->{$dev}->{transmit};
2605
2606         if ($full) {
2607             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2608             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2609         }
2610
2611     }
2612
2613     my $ctime = gettimeofday;
2614
2615     foreach my $vmid (keys %$list) {
2616
2617         my $d = $res->{$vmid};
2618         my $pid = $d->{pid};
2619         next if !$pid;
2620
2621         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2622         next if !$pstat; # not running
2623
2624         my $used = $pstat->{utime} + $pstat->{stime};
2625
2626         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2627
2628         if ($pstat->{vsize}) {
2629             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2630         }
2631
2632         my $old = $last_proc_pid_stat->{$pid};
2633         if (!$old) {
2634             $last_proc_pid_stat->{$pid} = {
2635                 time => $ctime,
2636                 used => $used,
2637                 cpu => 0,
2638             };
2639             next;
2640         }
2641
2642         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2643
2644         if ($dtime > 1000) {
2645             my $dutime = $used -  $old->{used};
2646
2647             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2648             $last_proc_pid_stat->{$pid} = {
2649                 time => $ctime,
2650                 used => $used,
2651                 cpu => $d->{cpu},
2652             };
2653         } else {
2654             $d->{cpu} = $old->{cpu};
2655         }
2656     }
2657
2658     return $res if !$full;
2659
2660     my $qmpclient = PVE::QMPClient->new();
2661
2662     my $ballooncb = sub {
2663         my ($vmid, $resp) = @_;
2664
2665         my $info = $resp->{'return'};
2666         return if !$info->{max_mem};
2667
2668         my $d = $res->{$vmid};
2669
2670         # use memory assigned to VM
2671         $d->{maxmem} = $info->{max_mem};
2672         $d->{balloon} = $info->{actual};
2673
2674         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2675             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2676             $d->{freemem} = $info->{free_mem};
2677         }
2678
2679         $d->{ballooninfo} = $info;
2680     };
2681
2682     my $blockstatscb = sub {
2683         my ($vmid, $resp) = @_;
2684         my $data = $resp->{'return'} || [];
2685         my $totalrdbytes = 0;
2686         my $totalwrbytes = 0;
2687
2688         for my $blockstat (@$data) {
2689             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2690             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2691
2692             $blockstat->{device} =~ s/drive-//;
2693             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2694         }
2695         $res->{$vmid}->{diskread} = $totalrdbytes;
2696         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2697     };
2698
2699     my $statuscb = sub {
2700         my ($vmid, $resp) = @_;
2701
2702         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2703         # this fails if ballon driver is not loaded, so this must be
2704         # the last commnand (following command are aborted if this fails).
2705         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2706
2707         my $status = 'unknown';
2708         if (!defined($status = $resp->{'return'}->{status})) {
2709             warn "unable to get VM status\n";
2710             return;
2711         }
2712
2713         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2714     };
2715
2716     foreach my $vmid (keys %$list) {
2717         next if $opt_vmid && ($vmid ne $opt_vmid);
2718         next if !$res->{$vmid}->{pid}; # not running
2719         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2720     }
2721
2722     $qmpclient->queue_execute(undef, 2);
2723
2724     foreach my $vmid (keys %$list) {
2725         next if $opt_vmid && ($vmid ne $opt_vmid);
2726         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2727     }
2728
2729     return $res;
2730 }
2731
2732 sub foreach_drive {
2733     my ($conf, $func, @param) = @_;
2734
2735     foreach my $ds (valid_drive_names()) {
2736         next if !defined($conf->{$ds});
2737
2738         my $drive = parse_drive($ds, $conf->{$ds});
2739         next if !$drive;
2740
2741         &$func($ds, $drive, @param);
2742     }
2743 }
2744
2745 sub foreach_volid {
2746     my ($conf, $func, @param) = @_;
2747
2748     my $volhash = {};
2749
2750     my $test_volid = sub {
2751         my ($volid, $is_cdrom) = @_;
2752
2753         return if !$volid;
2754
2755         $volhash->{$volid} = $is_cdrom || 0;
2756     };
2757
2758     foreach_drive($conf, sub {
2759         my ($ds, $drive) = @_;
2760         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2761     });
2762
2763     foreach my $snapname (keys %{$conf->{snapshots}}) {
2764         my $snap = $conf->{snapshots}->{$snapname};
2765         &$test_volid($snap->{vmstate}, 0);
2766         foreach_drive($snap, sub {
2767             my ($ds, $drive) = @_;
2768             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2769         });
2770     }
2771
2772     foreach my $volid (keys %$volhash) {
2773         &$func($volid, $volhash->{$volid}, @param);
2774     }
2775 }
2776
2777 sub vga_conf_has_spice {
2778     my ($vga) = @_;
2779
2780     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2781
2782     return $1 || 1;
2783 }
2784
2785 sub config_to_command {
2786     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2787
2788     my $cmd = [];
2789     my $globalFlags = [];
2790     my $machineFlags = [];
2791     my $rtcFlags = [];
2792     my $cpuFlags = [];
2793     my $devices = [];
2794     my $pciaddr = '';
2795     my $bridges = {};
2796     my $kvmver = kvm_user_version();
2797     my $vernum = 0; # unknown
2798     my $ostype = $conf->{ostype};
2799     my $winversion = windows_version($ostype);
2800
2801     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2802         $vernum = $1*1000000+$2*1000;
2803     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2804         $vernum = $1*1000000+$2*1000+$3;
2805     }
2806
2807     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2808
2809     my $have_ovz = -f '/proc/vz/vestat';
2810
2811     my $q35 = machine_type_is_q35($conf);
2812     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2813     my $machine_type = $forcemachine || $conf->{machine};
2814     my $use_old_bios_files = undef;
2815     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2816
2817     my $cpuunits = defined($conf->{cpuunits}) ?
2818             $conf->{cpuunits} : $defaults->{cpuunits};
2819
2820     push @$cmd, '/usr/bin/kvm';
2821
2822     push @$cmd, '-id', $vmid;
2823
2824     my $use_virtio = 0;
2825
2826     my $qmpsocket = qmp_socket($vmid);
2827     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2828     push @$cmd, '-mon', "chardev=qmp,mode=control";
2829
2830
2831     push @$cmd, '-pidfile' , pidfile_name($vmid);
2832
2833     push @$cmd, '-daemonize';
2834
2835     if ($conf->{smbios1}) {
2836         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2837     }
2838
2839     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2840         my $ovmfbase;
2841
2842         # prefer the OVMF_CODE variant
2843         if (-f $OVMF_CODE) {
2844             $ovmfbase = $OVMF_CODE;
2845         } elsif (-f $OVMF_IMG) {
2846             $ovmfbase = $OVMF_IMG;
2847         }
2848
2849         die "no uefi base img found\n" if !$ovmfbase;
2850         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmfbase";
2851
2852         if (defined($conf->{efidisk0}) && ($ovmfbase eq $OVMF_CODE)) {
2853             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $conf->{efidisk0});
2854             my $format = $d->{format} // 'raw';
2855             my $path;
2856             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2857             if ($storeid) {
2858                 $path = PVE::Storage::path($storecfg, $d->{file});
2859                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2860                 $format = qemu_img_format($scfg, $volname);
2861             } else {
2862                 $path = $d->{file};
2863                 $format = "raw";
2864             }
2865             push @$cmd, '-drive', "if=pflash,unit=1,id=drive-efidisk0,format=$format,file=$path";
2866         } elsif ($ovmfbase eq $OVMF_CODE) {
2867             warn "using uefi without permanent efivars disk\n";
2868             my $ovmfvar_dst = "/tmp/$vmid-ovmf.fd";
2869             PVE::Tools::file_copy($OVMF_VARS, $ovmfvar_dst, 256*1024);
2870             push @$cmd, '-drive', "if=pflash,unit=1,format=raw,file=$ovmfvar_dst";
2871         } else {
2872             # if the base img is not OVMF_CODE, we do not have to bother
2873             # to create/use a vars image, since it will not be used anyway
2874             # this can only happen if someone manually deletes the OVMF_CODE image
2875             # or has an old pve-qemu-kvm version installed.
2876             # both should not happen, but we ignore it here
2877         }
2878     }
2879
2880
2881     # add usb controllers
2882     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2883     push @$devices, @usbcontrollers if @usbcontrollers;
2884     my $vga = $conf->{vga};
2885
2886     my $qxlnum = vga_conf_has_spice($vga);
2887     $vga = 'qxl' if $qxlnum;
2888
2889     if (!$vga) {
2890         $vga = $winversion >= 6 ? 'std' : 'cirrus';
2891     }
2892
2893     # enable absolute mouse coordinates (needed by vnc)
2894     my $tablet;
2895     if (defined($conf->{tablet})) {
2896         $tablet = $conf->{tablet};
2897     } else {
2898         $tablet = $defaults->{tablet};
2899         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2900         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2901     }
2902
2903     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2904
2905     my $kvm_off = 0;
2906     my $gpu_passthrough;
2907
2908     # host pci devices
2909     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2910         my $d = parse_hostpci($conf->{"hostpci$i"});
2911         next if !$d;
2912
2913         my $pcie = $d->{pcie};
2914         if($pcie){
2915             die "q35 machine model is not enabled" if !$q35;
2916             $pciaddr = print_pcie_addr("hostpci$i");
2917         }else{
2918             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2919         }
2920
2921         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2922         my $romfile = $d->{romfile};
2923
2924         my $xvga = '';
2925         if ($d->{'x-vga'}) {
2926             $xvga = ',x-vga=on';
2927             $kvm_off = 1;
2928             $vga = 'none';
2929             $gpu_passthrough = 1;
2930
2931             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2932                 $xvga = "";
2933             }
2934         }
2935         my $pcidevices = $d->{pciid};
2936         my $multifunction = 1 if @$pcidevices > 1;
2937
2938         my $j=0;
2939         foreach my $pcidevice (@$pcidevices) {
2940
2941             my $id = "hostpci$i";
2942             $id .= ".$j" if $multifunction;
2943             my $addr = $pciaddr;
2944             $addr .= ".$j" if $multifunction;
2945             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2946
2947             if($j == 0){
2948                 $devicestr .= "$rombar$xvga";
2949                 $devicestr .= ",multifunction=on" if $multifunction;
2950                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
2951             }
2952
2953             push @$devices, '-device', $devicestr;
2954             $j++;
2955         }
2956     }
2957
2958     # usb devices
2959     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
2960     push @$devices, @usbdevices if @usbdevices;
2961     # serial devices
2962     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2963         if (my $path = $conf->{"serial$i"}) {
2964             if ($path eq 'socket') {
2965                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2966                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2967                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2968             } else {
2969                 die "no such serial device\n" if ! -c $path;
2970                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2971                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2972             }
2973         }
2974     }
2975
2976     # parallel devices
2977     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2978         if (my $path = $conf->{"parallel$i"}) {
2979             die "no such parallel device\n" if ! -c $path;
2980             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2981             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2982             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2983         }
2984     }
2985
2986     my $vmname = $conf->{name} || "vm$vmid";
2987
2988     push @$cmd, '-name', $vmname;
2989
2990     my $sockets = 1;
2991     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2992     $sockets = $conf->{sockets} if  $conf->{sockets};
2993
2994     my $cores = $conf->{cores} || 1;
2995
2996     my $maxcpus = $sockets * $cores;
2997
2998     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2999
3000     my $allowed_vcpus = $cpuinfo->{cpus};
3001
3002     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3003         if ($allowed_vcpus < $maxcpus);
3004
3005     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3006
3007         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3008         for (my $i = 2; $i <= $vcpus; $i++)  {
3009             my $cpustr = print_cpu_device($conf,$i);
3010             push @$cmd, '-device', $cpustr;
3011         }
3012
3013     } else {
3014
3015         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3016     }
3017     push @$cmd, '-nodefaults';
3018
3019     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3020
3021     my $bootindex_hash = {};
3022     my $i = 1;
3023     foreach my $o (split(//, $bootorder)) {
3024         $bootindex_hash->{$o} = $i*100;
3025         $i++;
3026     }
3027
3028     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3029
3030     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3031
3032     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3033
3034     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3035
3036     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3037         my $socket = vnc_socket($vmid);
3038         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3039     } else {
3040         push @$cmd, '-nographic';
3041     }
3042
3043     # time drift fix
3044     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3045
3046     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3047     my $useLocaltime = $conf->{localtime};
3048
3049     if ($winversion >= 5) { # windows
3050         $useLocaltime = 1 if !defined($conf->{localtime});
3051
3052         # use time drift fix when acpi is enabled
3053         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3054             $tdf = 1 if !defined($conf->{tdf});
3055         }
3056     }
3057
3058     if ($winversion >= 6) {
3059         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3060         push @$cmd, '-no-hpet';
3061     }
3062
3063     push @$rtcFlags, 'driftfix=slew' if $tdf;
3064
3065     if ($nokvm) {
3066         push @$machineFlags, 'accel=tcg';
3067     } else {
3068         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3069     }
3070
3071     if ($machine_type) {
3072         push @$machineFlags, "type=${machine_type}";
3073     }
3074
3075     if ($conf->{startdate}) {
3076         push @$rtcFlags, "base=$conf->{startdate}";
3077     } elsif ($useLocaltime) {
3078         push @$rtcFlags, 'base=localtime';
3079     }
3080
3081     my $cpu = $nokvm ? "qemu64" : "kvm64";
3082     if (my $cputype = $conf->{cpu}) {
3083         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3084             or die "Cannot parse cpu description: $cputype\n";
3085         $cpu = $cpuconf->{cputype};
3086         $kvm_off = 1 if $cpuconf->{hidden};
3087     }
3088
3089     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3090
3091     push @$cpuFlags , '-x2apic'
3092         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3093
3094     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3095
3096     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3097
3098     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3099
3100         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3101         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3102     }
3103
3104     add_hyperv_enlighments($cpuFlags, $winversion, $machine_type, $kvmver, $nokvm, $conf->{bios}, $gpu_passthrough);
3105
3106     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3107
3108     push @$cpuFlags, 'kvm=off' if $kvm_off;
3109
3110     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3111         die "internal error"; # should not happen
3112
3113     push @$cpuFlags, "vendor=${cpu_vendor}"
3114         if $cpu_vendor ne 'default';
3115
3116     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3117
3118     push @$cmd, '-cpu', $cpu;
3119
3120     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3121
3122     push @$cmd, '-S' if $conf->{freeze};
3123
3124     # set keyboard layout
3125     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3126     push @$cmd, '-k', $kb if $kb;
3127
3128     # enable sound
3129     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3130     #push @$cmd, '-soundhw', 'es1370';
3131     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3132
3133     if($conf->{agent}) {
3134         my $qgasocket = qmp_socket($vmid, 1);
3135         my $pciaddr = print_pci_addr("qga0", $bridges);
3136         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3137         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3138         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3139     }
3140
3141     my $spice_port;
3142
3143     if ($qxlnum) {
3144         if ($qxlnum > 1) {
3145             if ($winversion){
3146                 for(my $i = 1; $i < $qxlnum; $i++){
3147                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3148                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3149                 }
3150             } else {
3151                 # assume other OS works like Linux
3152                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3153                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3154             }
3155         }
3156
3157         my $pciaddr = print_pci_addr("spice", $bridges);
3158
3159         my $nodename = PVE::INotify::nodename();
3160         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3161         $spice_port = PVE::Tools::next_spice_port($pfamily);
3162
3163         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=HIGH,seamless-migration=on";
3164
3165         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3166         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3167         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3168     }
3169
3170     # enable balloon by default, unless explicitly disabled
3171     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3172         $pciaddr = print_pci_addr("balloon0", $bridges);
3173         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3174     }
3175
3176     if ($conf->{watchdog}) {
3177         my $wdopts = parse_watchdog($conf->{watchdog});
3178         $pciaddr = print_pci_addr("watchdog", $bridges);
3179         my $watchdog = $wdopts->{model} || 'i6300esb';
3180         push @$devices, '-device', "$watchdog$pciaddr";
3181         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3182     }
3183
3184     my $vollist = [];
3185     my $scsicontroller = {};
3186     my $ahcicontroller = {};
3187     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3188
3189     # Add iscsi initiator name if available
3190     if (my $initiator = get_initiator_name()) {
3191         push @$devices, '-iscsi', "initiator-name=$initiator";
3192     }
3193
3194     foreach_drive($conf, sub {
3195         my ($ds, $drive) = @_;
3196
3197         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3198             push @$vollist, $drive->{file};
3199         }
3200
3201         $use_virtio = 1 if $ds =~ m/^virtio/;
3202
3203         if (drive_is_cdrom ($drive)) {
3204             if ($bootindex_hash->{d}) {
3205                 $drive->{bootindex} = $bootindex_hash->{d};
3206                 $bootindex_hash->{d} += 1;
3207             }
3208         } else {
3209             if ($bootindex_hash->{c}) {
3210                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3211                 $bootindex_hash->{c} += 1;
3212             }
3213         }
3214
3215         if($drive->{interface} eq 'virtio'){
3216            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3217         }
3218
3219         if ($drive->{interface} eq 'scsi') {
3220
3221             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3222
3223             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3224             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3225
3226             my $iothread = '';
3227             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3228                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3229                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3230             } elsif ($drive->{iothread}) {
3231                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3232             }
3233
3234             my $queues = '';
3235             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3236                 $queues = ",num_queues=$drive->{queues}";
3237             }
3238
3239             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3240             $scsicontroller->{$controller}=1;
3241         }
3242
3243         if ($drive->{interface} eq 'sata') {
3244            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3245            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3246            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3247            $ahcicontroller->{$controller}=1;
3248         }
3249
3250         if ($drive->{interface} eq 'efidisk') {
3251             # this will be added somewhere else
3252             return;
3253         }
3254
3255         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3256         push @$devices, '-drive',$drive_cmd;
3257         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3258     });
3259
3260     for (my $i = 0; $i < $MAX_NETS; $i++) {
3261          next if !$conf->{"net$i"};
3262          my $d = parse_net($conf->{"net$i"});
3263          next if !$d;
3264
3265          $use_virtio = 1 if $d->{model} eq 'virtio';
3266
3267          if ($bootindex_hash->{n}) {
3268             $d->{bootindex} = $bootindex_hash->{n};
3269             $bootindex_hash->{n} += 1;
3270          }
3271
3272          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3273          push @$devices, '-netdev', $netdevfull;
3274
3275          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3276          push @$devices, '-device', $netdevicefull;
3277     }
3278
3279     if (!$q35) {
3280         # add pci bridges
3281         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3282            $bridges->{1} = 1;
3283            $bridges->{2} = 1;
3284         }
3285
3286         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3287
3288         while (my ($k, $v) = each %$bridges) {
3289             $pciaddr = print_pci_addr("pci.$k");
3290             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3291         }
3292     }
3293
3294     # add custom args
3295     if ($conf->{args}) {
3296         my $aa = PVE::Tools::split_args($conf->{args});
3297         push @$cmd, @$aa;
3298     }
3299
3300     push @$cmd, @$devices;
3301     push @$cmd, '-rtc', join(',', @$rtcFlags)
3302         if scalar(@$rtcFlags);
3303     push @$cmd, '-machine', join(',', @$machineFlags)
3304         if scalar(@$machineFlags);
3305     push @$cmd, '-global', join(',', @$globalFlags)
3306         if scalar(@$globalFlags);
3307
3308     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3309 }
3310
3311 sub vnc_socket {
3312     my ($vmid) = @_;
3313     return "${var_run_tmpdir}/$vmid.vnc";
3314 }
3315
3316 sub spice_port {
3317     my ($vmid) = @_;
3318
3319     my $res = vm_mon_cmd($vmid, 'query-spice');
3320
3321     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3322 }
3323
3324 sub qmp_socket {
3325     my ($vmid, $qga) = @_;
3326     my $sockettype = $qga ? 'qga' : 'qmp';
3327     return "${var_run_tmpdir}/$vmid.$sockettype";
3328 }
3329
3330 sub pidfile_name {
3331     my ($vmid) = @_;
3332     return "${var_run_tmpdir}/$vmid.pid";
3333 }
3334
3335 sub vm_devices_list {
3336     my ($vmid) = @_;
3337
3338     my $res = vm_mon_cmd($vmid, 'query-pci');
3339     my $devices = {};
3340     foreach my $pcibus (@$res) {
3341         foreach my $device (@{$pcibus->{devices}}) {
3342             next if !$device->{'qdev_id'};
3343             if ($device->{'pci_bridge'}) {
3344                 $devices->{$device->{'qdev_id'}} = 1;
3345                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3346                     next if !$bridge_device->{'qdev_id'};
3347                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3348                     $devices->{$device->{'qdev_id'}}++;
3349                 }
3350             } else {
3351                 $devices->{$device->{'qdev_id'}} = 1;
3352             }
3353         }
3354     }
3355
3356     my $resblock = vm_mon_cmd($vmid, 'query-block');
3357     foreach my $block (@$resblock) {
3358         if($block->{device} =~ m/^drive-(\S+)/){
3359                 $devices->{$1} = 1;
3360         }
3361     }
3362
3363     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3364     foreach my $mice (@$resmice) {
3365         if ($mice->{name} eq 'QEMU HID Tablet') {
3366             $devices->{tablet} = 1;
3367             last;
3368         }
3369     }
3370
3371     # for usb devices there is no query-usb
3372     # but we can iterate over the entries in
3373     # qom-list path=/machine/peripheral
3374     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3375     foreach my $per (@$resperipheral) {
3376         if ($per->{name} =~ m/^usb\d+$/) {
3377             $devices->{$per->{name}} = 1;
3378         }
3379     }
3380
3381     return $devices;
3382 }
3383
3384 sub vm_deviceplug {
3385     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3386
3387     my $q35 = machine_type_is_q35($conf);
3388
3389     my $devices_list = vm_devices_list($vmid);
3390     return 1 if defined($devices_list->{$deviceid});
3391
3392     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3393
3394     if ($deviceid eq 'tablet') {
3395
3396         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3397
3398     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3399
3400         die "usb hotplug currently not reliable\n";
3401         # since we can't reliably hot unplug all added usb devices
3402         # and usb passthrough disables live migration
3403         # we disable usb hotplugging for now
3404         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3405
3406     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3407
3408         qemu_iothread_add($vmid, $deviceid, $device);
3409
3410         qemu_driveadd($storecfg, $vmid, $device);
3411         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3412
3413         qemu_deviceadd($vmid, $devicefull);
3414         eval { qemu_deviceaddverify($vmid, $deviceid); };
3415         if (my $err = $@) {
3416             eval { qemu_drivedel($vmid, $deviceid); };
3417             warn $@ if $@;
3418             die $err;
3419         }
3420
3421     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3422
3423
3424         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3425         my $pciaddr = print_pci_addr($deviceid);
3426         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3427
3428         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3429
3430         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3431             qemu_iothread_add($vmid, $deviceid, $device);
3432             $devicefull .= ",iothread=iothread-$deviceid";
3433         }
3434
3435         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3436             $devicefull .= ",num_queues=$device->{queues}";
3437         }
3438
3439         qemu_deviceadd($vmid, $devicefull);
3440         qemu_deviceaddverify($vmid, $deviceid);
3441
3442     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3443
3444         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3445         qemu_driveadd($storecfg, $vmid, $device);
3446
3447         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3448         eval { qemu_deviceadd($vmid, $devicefull); };
3449         if (my $err = $@) {
3450             eval { qemu_drivedel($vmid, $deviceid); };
3451             warn $@ if $@;
3452             die $err;
3453         }
3454
3455     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3456
3457         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3458
3459         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3460         my $use_old_bios_files = undef;
3461         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3462
3463         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3464         qemu_deviceadd($vmid, $netdevicefull);
3465         eval { qemu_deviceaddverify($vmid, $deviceid); };
3466         if (my $err = $@) {
3467             eval { qemu_netdevdel($vmid, $deviceid); };
3468             warn $@ if $@;
3469             die $err;
3470         }
3471
3472     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3473
3474         my $bridgeid = $2;
3475         my $pciaddr = print_pci_addr($deviceid);
3476         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3477
3478         qemu_deviceadd($vmid, $devicefull);
3479         qemu_deviceaddverify($vmid, $deviceid);
3480
3481     } else {
3482         die "can't hotplug device '$deviceid'\n";
3483     }
3484
3485     return 1;
3486 }
3487
3488 # fixme: this should raise exceptions on error!
3489 sub vm_deviceunplug {
3490     my ($vmid, $conf, $deviceid) = @_;
3491
3492     my $devices_list = vm_devices_list($vmid);
3493     return 1 if !defined($devices_list->{$deviceid});
3494
3495     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3496
3497     if ($deviceid eq 'tablet') {
3498
3499         qemu_devicedel($vmid, $deviceid);
3500
3501     } elsif ($deviceid =~ m/^usb\d+$/) {
3502
3503         die "usb hotplug currently not reliable\n";
3504         # when unplugging usb devices this way,
3505         # there may be remaining usb controllers/hubs
3506         # so we disable it for now
3507         qemu_devicedel($vmid, $deviceid);
3508         qemu_devicedelverify($vmid, $deviceid);
3509
3510     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3511
3512         qemu_devicedel($vmid, $deviceid);
3513         qemu_devicedelverify($vmid, $deviceid);
3514         qemu_drivedel($vmid, $deviceid);
3515         qemu_iothread_del($conf, $vmid, $deviceid);
3516
3517     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3518
3519         qemu_devicedel($vmid, $deviceid);
3520         qemu_devicedelverify($vmid, $deviceid);
3521         qemu_iothread_del($conf, $vmid, $deviceid);
3522
3523     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3524
3525         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3526         my $device = parse_drive($deviceid, $conf->{$deviceid});
3527         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3528
3529         qemu_devicedel($vmid, $deviceid);
3530         qemu_drivedel($vmid, $deviceid);
3531         qemu_deletescsihw($conf, $vmid, $deviceid);
3532
3533     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3534
3535         qemu_devicedel($vmid, $deviceid);
3536         qemu_devicedelverify($vmid, $deviceid);
3537         qemu_netdevdel($vmid, $deviceid);
3538
3539     } else {
3540         die "can't unplug device '$deviceid'\n";
3541     }
3542
3543     return 1;
3544 }
3545
3546 sub qemu_deviceadd {
3547     my ($vmid, $devicefull) = @_;
3548
3549     $devicefull = "driver=".$devicefull;
3550     my %options =  split(/[=,]/, $devicefull);
3551
3552     vm_mon_cmd($vmid, "device_add" , %options);
3553 }
3554
3555 sub qemu_devicedel {
3556     my ($vmid, $deviceid) = @_;
3557
3558     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3559 }
3560
3561 sub qemu_iothread_add {
3562     my($vmid, $deviceid, $device) = @_;
3563
3564     if ($device->{iothread}) {
3565         my $iothreads = vm_iothreads_list($vmid);
3566         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3567     }
3568 }
3569
3570 sub qemu_iothread_del {
3571     my($conf, $vmid, $deviceid) = @_;
3572
3573     my $device = parse_drive($deviceid, $conf->{$deviceid});
3574     if ($device->{iothread}) {
3575         my $iothreads = vm_iothreads_list($vmid);
3576         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3577     }
3578 }
3579
3580 sub qemu_objectadd {
3581     my($vmid, $objectid, $qomtype) = @_;
3582
3583     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3584
3585     return 1;
3586 }
3587
3588 sub qemu_objectdel {
3589     my($vmid, $objectid) = @_;
3590
3591     vm_mon_cmd($vmid, "object-del", id => $objectid);
3592
3593     return 1;
3594 }
3595
3596 sub qemu_driveadd {
3597     my ($storecfg, $vmid, $device) = @_;
3598
3599     my $drive = print_drive_full($storecfg, $vmid, $device);
3600     $drive =~ s/\\/\\\\/g;
3601     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3602
3603     # If the command succeeds qemu prints: "OK"
3604     return 1 if $ret =~ m/OK/s;
3605
3606     die "adding drive failed: $ret\n";
3607 }
3608
3609 sub qemu_drivedel {
3610     my($vmid, $deviceid) = @_;
3611
3612     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3613     $ret =~ s/^\s+//;
3614
3615     return 1 if $ret eq "";
3616
3617     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3618     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3619
3620     die "deleting drive $deviceid failed : $ret\n";
3621 }
3622
3623 sub qemu_deviceaddverify {
3624     my ($vmid, $deviceid) = @_;
3625
3626     for (my $i = 0; $i <= 5; $i++) {
3627          my $devices_list = vm_devices_list($vmid);
3628          return 1 if defined($devices_list->{$deviceid});
3629          sleep 1;
3630     }
3631
3632     die "error on hotplug device '$deviceid'\n";
3633 }
3634
3635
3636 sub qemu_devicedelverify {
3637     my ($vmid, $deviceid) = @_;
3638
3639     # need to verify that the device is correctly removed as device_del
3640     # is async and empty return is not reliable
3641
3642     for (my $i = 0; $i <= 5; $i++) {
3643          my $devices_list = vm_devices_list($vmid);
3644          return 1 if !defined($devices_list->{$deviceid});
3645          sleep 1;
3646     }
3647
3648     die "error on hot-unplugging device '$deviceid'\n";
3649 }
3650
3651 sub qemu_findorcreatescsihw {
3652     my ($storecfg, $conf, $vmid, $device) = @_;
3653
3654     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3655
3656     my $scsihwid="$controller_prefix$controller";
3657     my $devices_list = vm_devices_list($vmid);
3658
3659     if(!defined($devices_list->{$scsihwid})) {
3660         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3661     }
3662
3663     return 1;
3664 }
3665
3666 sub qemu_deletescsihw {
3667     my ($conf, $vmid, $opt) = @_;
3668
3669     my $device = parse_drive($opt, $conf->{$opt});
3670
3671     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3672         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3673         return 1;
3674     }
3675
3676     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3677
3678     my $devices_list = vm_devices_list($vmid);
3679     foreach my $opt (keys %{$devices_list}) {
3680         if (PVE::QemuServer::is_valid_drivename($opt)) {
3681             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3682             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3683                 return 1;
3684             }
3685         }
3686     }
3687
3688     my $scsihwid="scsihw$controller";
3689
3690     vm_deviceunplug($vmid, $conf, $scsihwid);
3691
3692     return 1;
3693 }
3694
3695 sub qemu_add_pci_bridge {
3696     my ($storecfg, $conf, $vmid, $device) = @_;
3697
3698     my $bridges = {};
3699
3700     my $bridgeid;
3701
3702     print_pci_addr($device, $bridges);
3703
3704     while (my ($k, $v) = each %$bridges) {
3705         $bridgeid = $k;
3706     }
3707     return 1 if !defined($bridgeid) || $bridgeid < 1;
3708
3709     my $bridge = "pci.$bridgeid";
3710     my $devices_list = vm_devices_list($vmid);
3711
3712     if (!defined($devices_list->{$bridge})) {
3713         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3714     }
3715
3716     return 1;
3717 }
3718
3719 sub qemu_set_link_status {
3720     my ($vmid, $device, $up) = @_;
3721
3722     vm_mon_cmd($vmid, "set_link", name => $device,
3723                up => $up ? JSON::true : JSON::false);
3724 }
3725
3726 sub qemu_netdevadd {
3727     my ($vmid, $conf, $device, $deviceid) = @_;
3728
3729     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3730     my %options =  split(/[=,]/, $netdev);
3731
3732     vm_mon_cmd($vmid, "netdev_add",  %options);
3733     return 1;
3734 }
3735
3736 sub qemu_netdevdel {
3737     my ($vmid, $deviceid) = @_;
3738
3739     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3740 }
3741
3742 sub qemu_usb_hotplug {
3743     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3744
3745     return if !$device;
3746
3747     # remove the old one first
3748     vm_deviceunplug($vmid, $conf, $deviceid);
3749
3750     # check if xhci controller is necessary and available
3751     if ($device->{usb3}) {
3752
3753         my $devicelist = vm_devices_list($vmid);
3754
3755         if (!$devicelist->{xhci}) {
3756             my $pciaddr = print_pci_addr("xhci");
3757             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3758         }
3759     }
3760     my $d = parse_usb_device($device->{host});
3761     $d->{usb3} = $device->{usb3};
3762
3763     # add the new one
3764     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3765 }
3766
3767 sub qemu_cpu_hotplug {
3768     my ($vmid, $conf, $vcpus) = @_;
3769
3770     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3771
3772     my $sockets = 1;
3773     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3774     $sockets = $conf->{sockets} if  $conf->{sockets};
3775     my $cores = $conf->{cores} || 1;
3776     my $maxcpus = $sockets * $cores;
3777
3778     $vcpus = $maxcpus if !$vcpus;
3779
3780     die "you can't add more vcpus than maxcpus\n"
3781         if $vcpus > $maxcpus;
3782
3783     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3784
3785     if ($vcpus < $currentvcpus) {
3786
3787         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3788
3789             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3790                 qemu_devicedel($vmid, "cpu$i");
3791                 my $retry = 0;
3792                 my $currentrunningvcpus = undef;
3793                 while (1) {
3794                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3795                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3796                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3797                     $retry++;
3798                     sleep 1;
3799                 }
3800                 #update conf after each succesfull cpu unplug
3801                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3802                 PVE::QemuConfig->write_config($vmid, $conf);
3803             }
3804         } else {
3805             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3806         }
3807
3808         return;
3809     }
3810
3811     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3812     die "vcpus in running vm does not match its configuration\n"
3813         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3814
3815     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3816
3817         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3818             my $cpustr = print_cpu_device($conf, $i);
3819             qemu_deviceadd($vmid, $cpustr);
3820
3821             my $retry = 0;
3822             my $currentrunningvcpus = undef;
3823             while (1) {
3824                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3825                 last if scalar(@{$currentrunningvcpus}) == $i;
3826                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3827                 sleep 1;
3828                 $retry++;
3829             }
3830             #update conf after each succesfull cpu hotplug
3831             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3832             PVE::QemuConfig->write_config($vmid, $conf);
3833         }
3834     } else {
3835
3836         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3837             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3838         }
3839     }
3840 }
3841
3842 sub qemu_block_set_io_throttle {
3843     my ($vmid, $deviceid,
3844         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3845         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3846         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3847         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3848
3849     return if !check_running($vmid) ;
3850
3851     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3852         bps => int($bps),
3853         bps_rd => int($bps_rd),
3854         bps_wr => int($bps_wr),
3855         iops => int($iops),
3856         iops_rd => int($iops_rd),
3857         iops_wr => int($iops_wr),
3858         bps_max => int($bps_max),
3859         bps_rd_max => int($bps_rd_max),
3860         bps_wr_max => int($bps_wr_max),
3861         iops_max => int($iops_max),
3862         iops_rd_max => int($iops_rd_max),
3863         iops_wr_max => int($iops_wr_max),
3864         bps_max_length => int($bps_max_length),
3865         bps_rd_max_length => int($bps_rd_max_length),
3866         bps_wr_max_length => int($bps_wr_max_length),
3867         iops_max_length => int($iops_max_length),
3868         iops_rd_max_length => int($iops_rd_max_length),
3869         iops_wr_max_length => int($iops_wr_max_length),
3870     );
3871
3872 }
3873
3874 # old code, only used to shutdown old VM after update
3875 sub __read_avail {
3876     my ($fh, $timeout) = @_;
3877
3878     my $sel = new IO::Select;
3879     $sel->add($fh);
3880
3881     my $res = '';
3882     my $buf;
3883
3884     my @ready;
3885     while (scalar (@ready = $sel->can_read($timeout))) {
3886         my $count;
3887         if ($count = $fh->sysread($buf, 8192)) {
3888             if ($buf =~ /^(.*)\(qemu\) $/s) {
3889                 $res .= $1;
3890                 last;
3891             } else {
3892                 $res .= $buf;
3893             }
3894         } else {
3895             if (!defined($count)) {
3896                 die "$!\n";
3897             }
3898             last;
3899         }
3900     }
3901
3902     die "monitor read timeout\n" if !scalar(@ready);
3903
3904     return $res;
3905 }
3906
3907 # old code, only used to shutdown old VM after update
3908 sub vm_monitor_command {
3909     my ($vmid, $cmdstr, $nocheck) = @_;
3910
3911     my $res;
3912
3913     eval {
3914         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3915
3916         my $sname = "${var_run_tmpdir}/$vmid.mon";
3917
3918         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3919             die "unable to connect to VM $vmid socket - $!\n";
3920
3921         my $timeout = 3;
3922
3923         # hack: migrate sometime blocks the monitor (when migrate_downtime
3924         # is set)
3925         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3926             $timeout = 60*60; # 1 hour
3927         }
3928
3929         # read banner;
3930         my $data = __read_avail($sock, $timeout);
3931
3932         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3933             die "got unexpected qemu monitor banner\n";
3934         }
3935
3936         my $sel = new IO::Select;
3937         $sel->add($sock);
3938
3939         if (!scalar(my @ready = $sel->can_write($timeout))) {
3940             die "monitor write error - timeout";
3941         }
3942
3943         my $fullcmd = "$cmdstr\r";
3944
3945         # syslog('info', "VM $vmid monitor command: $cmdstr");
3946
3947         my $b;
3948         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3949             die "monitor write error - $!";
3950         }
3951
3952         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3953
3954         $timeout = 20;
3955
3956         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3957             $timeout = 60*60; # 1 hour
3958         } elsif ($cmdstr =~ m/^(eject|change)/) {
3959             $timeout = 60; # note: cdrom mount command is slow
3960         }
3961         if ($res = __read_avail($sock, $timeout)) {
3962
3963             my @lines = split("\r?\n", $res);
3964
3965             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3966
3967             $res = join("\n", @lines);
3968             $res .= "\n";
3969         }
3970     };
3971
3972     my $err = $@;
3973
3974     if ($err) {
3975         syslog("err", "VM $vmid monitor command failed - $err");
3976         die $err;
3977     }
3978
3979     return $res;
3980 }
3981
3982 sub qemu_block_resize {
3983     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3984
3985     my $running = check_running($vmid);
3986
3987     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3988
3989     return if !$running;
3990
3991     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3992
3993 }
3994
3995 sub qemu_volume_snapshot {
3996     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3997
3998     my $running = check_running($vmid);
3999
4000     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4001         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4002     } else {
4003         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4004     }
4005 }
4006
4007 sub qemu_volume_snapshot_delete {
4008     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4009
4010     my $running = check_running($vmid);
4011
4012     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4013         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4014     } else {
4015         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4016     }
4017 }
4018
4019 sub set_migration_caps {
4020     my ($vmid) = @_;
4021
4022     my $cap_ref = [];
4023
4024     my $enabled_cap = {
4025         "auto-converge" => 1,
4026         "xbzrle" => 1,
4027         "x-rdma-pin-all" => 0,
4028         "zero-blocks" => 0,
4029         "compress" => 0
4030     };
4031
4032     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4033
4034     for my $supported_capability (@$supported_capabilities) {
4035         push @$cap_ref, {
4036             capability => $supported_capability->{capability},
4037             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4038         };
4039     }
4040
4041     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4042 }
4043
4044 my $fast_plug_option = {
4045     'lock' => 1,
4046     'name' => 1,
4047     'onboot' => 1,
4048     'shares' => 1,
4049     'startup' => 1,
4050     'description' => 1,
4051     'protection' => 1,
4052 };
4053
4054 # hotplug changes in [PENDING]
4055 # $selection hash can be used to only apply specified options, for
4056 # example: { cores => 1 } (only apply changed 'cores')
4057 # $errors ref is used to return error messages
4058 sub vmconfig_hotplug_pending {
4059     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4060
4061     my $defaults = load_defaults();
4062
4063     # commit values which do not have any impact on running VM first
4064     # Note: those option cannot raise errors, we we do not care about
4065     # $selection and always apply them.
4066
4067     my $add_error = sub {
4068         my ($opt, $msg) = @_;
4069         $errors->{$opt} = "hotplug problem - $msg";
4070     };
4071
4072     my $changes = 0;
4073     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4074         if ($fast_plug_option->{$opt}) {
4075             $conf->{$opt} = $conf->{pending}->{$opt};
4076             delete $conf->{pending}->{$opt};
4077             $changes = 1;
4078         }
4079     }
4080
4081     if ($changes) {
4082         PVE::QemuConfig->write_config($vmid, $conf);
4083         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4084     }
4085
4086     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4087
4088     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4089     while (my ($opt, $force) = each %$pending_delete_hash) {
4090         next if $selection && !$selection->{$opt};
4091         eval {
4092             if ($opt eq 'hotplug') {
4093                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4094             } elsif ($opt eq 'tablet') {
4095                 die "skip\n" if !$hotplug_features->{usb};
4096                 if ($defaults->{tablet}) {
4097                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4098                 } else {
4099                     vm_deviceunplug($vmid, $conf, $opt);
4100                 }
4101             } elsif ($opt =~ m/^usb\d+/) {
4102                 die "skip\n";
4103                 # since we cannot reliably hot unplug usb devices
4104                 # we are disabling it
4105                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4106                 vm_deviceunplug($vmid, $conf, $opt);
4107             } elsif ($opt eq 'vcpus') {
4108                 die "skip\n" if !$hotplug_features->{cpu};
4109                 qemu_cpu_hotplug($vmid, $conf, undef);
4110             } elsif ($opt eq 'balloon') {
4111                 # enable balloon device is not hotpluggable
4112                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4113             } elsif ($fast_plug_option->{$opt}) {
4114                 # do nothing
4115             } elsif ($opt =~ m/^net(\d+)$/) {
4116                 die "skip\n" if !$hotplug_features->{network};
4117                 vm_deviceunplug($vmid, $conf, $opt);
4118             } elsif (is_valid_drivename($opt)) {
4119                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4120                 vm_deviceunplug($vmid, $conf, $opt);
4121                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4122             } elsif ($opt =~ m/^memory$/) {
4123                 die "skip\n" if !$hotplug_features->{memory};
4124                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4125             } elsif ($opt eq 'cpuunits') {
4126                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4127             } elsif ($opt eq 'cpulimit') {
4128                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4129             } else {
4130                 die "skip\n";
4131             }
4132         };
4133         if (my $err = $@) {
4134             &$add_error($opt, $err) if $err ne "skip\n";
4135         } else {
4136             # save new config if hotplug was successful
4137             delete $conf->{$opt};
4138             vmconfig_undelete_pending_option($conf, $opt);
4139             PVE::QemuConfig->write_config($vmid, $conf);
4140             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4141         }
4142     }
4143
4144     foreach my $opt (keys %{$conf->{pending}}) {
4145         next if $selection && !$selection->{$opt};
4146         my $value = $conf->{pending}->{$opt};
4147         eval {
4148             if ($opt eq 'hotplug') {
4149                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4150             } elsif ($opt eq 'tablet') {
4151                 die "skip\n" if !$hotplug_features->{usb};
4152                 if ($value == 1) {
4153                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4154                 } elsif ($value == 0) {
4155                     vm_deviceunplug($vmid, $conf, $opt);
4156                 }
4157             } elsif ($opt =~ m/^usb\d+$/) {
4158                 die "skip\n";
4159                 # since we cannot reliably hot unplug usb devices
4160                 # we are disabling it
4161                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4162                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4163                 die "skip\n" if !$d;
4164                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4165             } elsif ($opt eq 'vcpus') {
4166                 die "skip\n" if !$hotplug_features->{cpu};
4167                 qemu_cpu_hotplug($vmid, $conf, $value);
4168             } elsif ($opt eq 'balloon') {
4169                 # enable/disable balloning device is not hotpluggable
4170                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4171                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4172                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4173
4174                 # allow manual ballooning if shares is set to zero
4175                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4176                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4177                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4178                 }
4179             } elsif ($opt =~ m/^net(\d+)$/) {
4180                 # some changes can be done without hotplug
4181                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4182                                     $vmid, $opt, $value);
4183             } elsif (is_valid_drivename($opt)) {
4184                 # some changes can be done without hotplug
4185                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4186                                      $vmid, $opt, $value, 1);
4187             } elsif ($opt =~ m/^memory$/) { #dimms
4188                 die "skip\n" if !$hotplug_features->{memory};
4189                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4190             } elsif ($opt eq 'cpuunits') {
4191                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4192             } elsif ($opt eq 'cpulimit') {
4193                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4194                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4195             } else {
4196                 die "skip\n";  # skip non-hot-pluggable options
4197             }
4198         };
4199         if (my $err = $@) {
4200             &$add_error($opt, $err) if $err ne "skip\n";
4201         } else {
4202             # save new config if hotplug was successful
4203             $conf->{$opt} = $value;
4204             delete $conf->{pending}->{$opt};
4205             PVE::QemuConfig->write_config($vmid, $conf);
4206             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4207         }
4208     }
4209 }
4210
4211 sub try_deallocate_drive {
4212     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4213
4214     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4215         my $volid = $drive->{file};
4216         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4217             my $sid = PVE::Storage::parse_volume_id($volid);
4218             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4219
4220             # check if the disk is really unused
4221             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4222                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4223             PVE::Storage::vdisk_free($storecfg, $volid);
4224             return 1;
4225         } else {
4226             # If vm is not owner of this disk remove from config
4227             return 1;
4228         }
4229     }
4230
4231     return undef;
4232 }
4233
4234 sub vmconfig_delete_or_detach_drive {
4235     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4236
4237     my $drive = parse_drive($opt, $conf->{$opt});
4238
4239     my $rpcenv = PVE::RPCEnvironment::get();
4240     my $authuser = $rpcenv->get_user();
4241
4242     if ($force) {
4243         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4244         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4245     } else {
4246         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4247     }
4248 }
4249
4250 sub vmconfig_apply_pending {
4251     my ($vmid, $conf, $storecfg) = @_;
4252
4253     # cold plug
4254
4255     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4256     while (my ($opt, $force) = each %$pending_delete_hash) {
4257         die "internal error" if $opt =~ m/^unused/;
4258         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4259         if (!defined($conf->{$opt})) {
4260             vmconfig_undelete_pending_option($conf, $opt);
4261             PVE::QemuConfig->write_config($vmid, $conf);
4262         } elsif (is_valid_drivename($opt)) {
4263             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4264             vmconfig_undelete_pending_option($conf, $opt);
4265             delete $conf->{$opt};
4266             PVE::QemuConfig->write_config($vmid, $conf);
4267         } else {
4268             vmconfig_undelete_pending_option($conf, $opt);
4269             delete $conf->{$opt};
4270             PVE::QemuConfig->write_config($vmid, $conf);
4271         }
4272     }
4273
4274     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4275
4276     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4277         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4278
4279         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4280             # skip if nothing changed
4281         } elsif (is_valid_drivename($opt)) {
4282             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4283                 if defined($conf->{$opt});
4284             $conf->{$opt} = $conf->{pending}->{$opt};
4285         } else {
4286             $conf->{$opt} = $conf->{pending}->{$opt};
4287         }
4288
4289         delete $conf->{pending}->{$opt};
4290         PVE::QemuConfig->write_config($vmid, $conf);
4291     }
4292 }
4293
4294 my $safe_num_ne = sub {
4295     my ($a, $b) = @_;
4296
4297     return 0 if !defined($a) && !defined($b);
4298     return 1 if !defined($a);
4299     return 1 if !defined($b);
4300
4301     return $a != $b;
4302 };
4303
4304 my $safe_string_ne = sub {
4305     my ($a, $b) = @_;
4306
4307     return 0 if !defined($a) && !defined($b);
4308     return 1 if !defined($a);
4309     return 1 if !defined($b);
4310
4311     return $a ne $b;
4312 };
4313
4314 sub vmconfig_update_net {
4315     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4316
4317     my $newnet = parse_net($value);
4318
4319     if ($conf->{$opt}) {
4320         my $oldnet = parse_net($conf->{$opt});
4321
4322         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4323             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4324             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4325             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4326
4327             # for non online change, we try to hot-unplug
4328             die "skip\n" if !$hotplug;
4329             vm_deviceunplug($vmid, $conf, $opt);
4330         } else {
4331
4332             die "internal error" if $opt !~ m/net(\d+)/;
4333             my $iface = "tap${vmid}i$1";
4334
4335             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4336                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4337                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4338                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4339                 PVE::Network::tap_unplug($iface);
4340                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4341             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4342                 # Rate can be applied on its own but any change above needs to
4343                 # include the rate in tap_plug since OVS resets everything.
4344                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4345             }
4346
4347             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4348                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4349             }
4350
4351             return 1;
4352         }
4353     }
4354
4355     if ($hotplug) {
4356         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4357     } else {
4358         die "skip\n";
4359     }
4360 }
4361
4362 sub vmconfig_update_disk {
4363     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4364
4365     # fixme: do we need force?
4366
4367     my $drive = parse_drive($opt, $value);
4368
4369     if ($conf->{$opt}) {
4370
4371         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4372
4373             my $media = $drive->{media} || 'disk';
4374             my $oldmedia = $old_drive->{media} || 'disk';
4375             die "unable to change media type\n" if $media ne $oldmedia;
4376
4377             if (!drive_is_cdrom($old_drive)) {
4378
4379                 if ($drive->{file} ne $old_drive->{file}) {
4380
4381                     die "skip\n" if !$hotplug;
4382
4383                     # unplug and register as unused
4384                     vm_deviceunplug($vmid, $conf, $opt);
4385                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4386
4387                 } else {
4388                     # update existing disk
4389
4390                     # skip non hotpluggable value
4391                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4392                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4393                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4394                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4395                         die "skip\n";
4396                     }
4397
4398                     # apply throttle
4399                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4400                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4401                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4402                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4403                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4404                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4405                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4406                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4407                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4408                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4409                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4410                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4411                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4412                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4413                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4414                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4415                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4416                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4417
4418                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4419                                                    ($drive->{mbps} || 0)*1024*1024,
4420                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4421                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4422                                                    $drive->{iops} || 0,
4423                                                    $drive->{iops_rd} || 0,
4424                                                    $drive->{iops_wr} || 0,
4425                                                    ($drive->{mbps_max} || 0)*1024*1024,
4426                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4427                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4428                                                    $drive->{iops_max} || 0,
4429                                                    $drive->{iops_rd_max} || 0,
4430                                                    $drive->{iops_wr_max} || 0,
4431                                                    $drive->{bps_max_length} || 1,
4432                                                    $drive->{bps_rd_max_length} || 1,
4433                                                    $drive->{bps_wr_max_length} || 1,
4434                                                    $drive->{iops_max_length} || 1,
4435                                                    $drive->{iops_rd_max_length} || 1,
4436                                                    $drive->{iops_wr_max_length} || 1);
4437
4438                     }
4439
4440                     return 1;
4441                 }
4442
4443             } else { # cdrom
4444
4445                 if ($drive->{file} eq 'none') {
4446                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4447                 } else {
4448                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4449                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4450                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4451                 }
4452
4453                 return 1;
4454             }
4455         }
4456     }
4457
4458     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4459     # hotplug new disks
4460     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4461     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4462 }
4463
4464 sub vm_start {
4465     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4466         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4467
4468     PVE::QemuConfig->lock_config($vmid, sub {
4469         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4470
4471         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4472
4473         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4474
4475         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4476
4477         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4478             vmconfig_apply_pending($vmid, $conf, $storecfg);
4479             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4480         }
4481
4482         my $defaults = load_defaults();
4483
4484         # set environment variable useful inside network script
4485         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4486
4487         my $local_volumes = {};
4488
4489         if ($targetstorage) {
4490             foreach_drive($conf, sub {
4491                 my ($ds, $drive) = @_;
4492
4493                 return if drive_is_cdrom($drive);
4494
4495                 my $volid = $drive->{file};
4496
4497                 return if !$volid;
4498
4499                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4500
4501                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4502                 return if $scfg->{shared};
4503                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4504             });
4505
4506             my $format = undef;
4507
4508             foreach my $opt (sort keys %$local_volumes) {
4509
4510                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4511                 my $drive = parse_drive($opt, $conf->{$opt});
4512
4513                 #if remote storage is specified, use default format
4514                 if ($targetstorage && $targetstorage ne "1") {
4515                     $storeid = $targetstorage;
4516                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4517                     $format = $defFormat;
4518                 } else {
4519                     #else we use same format than original
4520                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4521                     $format = qemu_img_format($scfg, $volid);
4522                 }
4523
4524                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4525                 my $newdrive = $drive;
4526                 $newdrive->{format} = $format;
4527                 $newdrive->{file} = $newvolid;
4528                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4529                 $local_volumes->{$opt} = $drivestr;
4530                 #pass drive to conf for command line
4531                 $conf->{$opt} = $drivestr;
4532             }
4533         }
4534
4535         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4536
4537         my $migrate_port = 0;
4538         my $migrate_uri;
4539         if ($statefile) {
4540             if ($statefile eq 'tcp') {
4541                 my $localip = "localhost";
4542                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4543                 my $nodename = PVE::INotify::nodename();
4544
4545                 if (!defined($migration_type)) {
4546                     if (defined($datacenterconf->{migration}->{type})) {
4547                         $migration_type = $datacenterconf->{migration}->{type};
4548                     } else {
4549                         $migration_type = 'secure';
4550                     }
4551                 }
4552
4553                 if ($migration_type eq 'insecure') {
4554                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4555                     if ($migrate_network_addr) {
4556                         $localip = $migrate_network_addr;
4557                     } else {
4558                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4559                     }
4560
4561                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4562                 }
4563
4564                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4565                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4566                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4567                 push @$cmd, '-incoming', $migrate_uri;
4568                 push @$cmd, '-S';
4569
4570             } elsif ($statefile eq 'unix') {
4571                 # should be default for secure migrations as a ssh TCP forward
4572                 # tunnel is not deterministic reliable ready and fails regurarly
4573                 # to set up in time, so use UNIX socket forwards
4574                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4575                 unlink $socket_addr;
4576
4577                 $migrate_uri = "unix:$socket_addr";
4578
4579                 push @$cmd, '-incoming', $migrate_uri;
4580                 push @$cmd, '-S';
4581
4582             } else {
4583                 push @$cmd, '-loadstate', $statefile;
4584             }
4585         } elsif ($paused) {
4586             push @$cmd, '-S';
4587         }
4588
4589         # host pci devices
4590         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4591           my $d = parse_hostpci($conf->{"hostpci$i"});
4592           next if !$d;
4593           my $pcidevices = $d->{pciid};
4594           foreach my $pcidevice (@$pcidevices) {
4595                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4596
4597                 my $info = pci_device_info("0000:$pciid");
4598                 die "IOMMU not present\n" if !check_iommu_support();
4599                 die "no pci device info for device '$pciid'\n" if !$info;
4600                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4601                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4602           }
4603         }
4604
4605         PVE::Storage::activate_volumes($storecfg, $vollist);
4606
4607         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4608             my $cmd = [];
4609             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4610             eval  { run_command($cmd); };
4611         }
4612
4613         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4614                                                   : $defaults->{cpuunits};
4615
4616         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4617
4618         my %properties = (
4619             Slice => 'qemu.slice',
4620             KillMode => 'none',
4621             CPUShares => $cpuunits
4622         );
4623
4624         if (my $cpulimit = $conf->{cpulimit}) {
4625             $properties{CPUQuota} = int($cpulimit * 100);
4626         }
4627         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4628
4629         if ($conf->{hugepages}) {
4630
4631             my $code = sub {
4632                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4633                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4634
4635                 PVE::QemuServer::Memory::hugepages_mount();
4636                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4637
4638                 eval  {
4639                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4640                     run_command($cmd, %run_params);
4641                 };
4642
4643                 if (my $err = $@) {
4644                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4645                     die $err;
4646                 }
4647
4648                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4649             };
4650             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4651
4652         } else {
4653             eval  {
4654                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4655                 run_command($cmd, %run_params);
4656             };
4657         }
4658
4659         if (my $err = $@) {
4660             # deactivate volumes if start fails
4661             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4662             die "start failed: $err";
4663         }
4664
4665         print "migration listens on $migrate_uri\n" if $migrate_uri;
4666
4667         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4668             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4669             warn $@ if $@;
4670         }
4671
4672         #start nbd server for storage migration
4673         if ($targetstorage) {
4674             my $nodename = PVE::INotify::nodename();
4675             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4676             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4677             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4678             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4679
4680             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4681
4682             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4683
4684             foreach my $opt (sort keys %$local_volumes) {
4685                 my $volid = $local_volumes->{$opt};
4686                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4687                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4688                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4689             }
4690         }
4691
4692         if ($migratedfrom) {
4693             eval {
4694                 set_migration_caps($vmid);
4695             };
4696             warn $@ if $@;
4697
4698             if ($spice_port) {
4699                 print "spice listens on port $spice_port\n";
4700                 if ($spice_ticket) {
4701                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4702                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4703                 }
4704             }
4705
4706         } else {
4707             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4708                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4709                     if $conf->{balloon};
4710             }
4711
4712             foreach my $opt (keys %$conf) {
4713                 next if $opt !~  m/^net\d+$/;
4714                 my $nicconf = parse_net($conf->{$opt});
4715                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4716             }
4717         }
4718
4719         vm_mon_cmd_nocheck($vmid, 'qom-set',
4720                     path => "machine/peripheral/balloon0",
4721                     property => "guest-stats-polling-interval",
4722                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4723
4724     });
4725 }
4726
4727 sub vm_mon_cmd {
4728     my ($vmid, $execute, %params) = @_;
4729
4730     my $cmd = { execute => $execute, arguments => \%params };
4731     vm_qmp_command($vmid, $cmd);
4732 }
4733
4734 sub vm_mon_cmd_nocheck {
4735     my ($vmid, $execute, %params) = @_;
4736
4737     my $cmd = { execute => $execute, arguments => \%params };
4738     vm_qmp_command($vmid, $cmd, 1);
4739 }
4740
4741 sub vm_qmp_command {
4742     my ($vmid, $cmd, $nocheck) = @_;
4743
4744     my $res;
4745
4746     my $timeout;
4747     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4748         $timeout = $cmd->{arguments}->{timeout};
4749         delete $cmd->{arguments}->{timeout};
4750     }
4751
4752     eval {
4753         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4754         my $sname = qmp_socket($vmid);
4755         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4756             my $qmpclient = PVE::QMPClient->new();
4757
4758             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4759         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4760             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4761                 if scalar(%{$cmd->{arguments}});
4762             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4763         } else {
4764             die "unable to open monitor socket\n";
4765         }
4766     };
4767     if (my $err = $@) {
4768         syslog("err", "VM $vmid qmp command failed - $err");
4769         die $err;
4770     }
4771
4772     return $res;
4773 }
4774
4775 sub vm_human_monitor_command {
4776     my ($vmid, $cmdline) = @_;
4777
4778     my $res;
4779
4780     my $cmd = {
4781         execute => 'human-monitor-command',
4782         arguments => { 'command-line' => $cmdline},
4783     };
4784
4785     return vm_qmp_command($vmid, $cmd);
4786 }
4787
4788 sub vm_commandline {
4789     my ($storecfg, $vmid) = @_;
4790
4791     my $conf = PVE::QemuConfig->load_config($vmid);
4792
4793     my $defaults = load_defaults();
4794
4795     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4796
4797     return PVE::Tools::cmd2string($cmd);
4798 }
4799
4800 sub vm_reset {
4801     my ($vmid, $skiplock) = @_;
4802
4803     PVE::QemuConfig->lock_config($vmid, sub {
4804
4805         my $conf = PVE::QemuConfig->load_config($vmid);
4806
4807         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4808
4809         vm_mon_cmd($vmid, "system_reset");
4810     });
4811 }
4812
4813 sub get_vm_volumes {
4814     my ($conf) = @_;
4815
4816     my $vollist = [];
4817     foreach_volid($conf, sub {
4818         my ($volid, $is_cdrom) = @_;
4819
4820         return if $volid =~ m|^/|;
4821
4822         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4823         return if !$sid;
4824
4825         push @$vollist, $volid;
4826     });
4827
4828     return $vollist;
4829 }
4830
4831 sub vm_stop_cleanup {
4832     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4833
4834     eval {
4835
4836         if (!$keepActive) {
4837             my $vollist = get_vm_volumes($conf);
4838             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4839         }
4840
4841         foreach my $ext (qw(mon qmp pid vnc qga)) {
4842             unlink "/var/run/qemu-server/${vmid}.$ext";
4843         }
4844
4845         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4846     };
4847     warn $@ if $@; # avoid errors - just warn
4848 }
4849
4850 # Note: use $nockeck to skip tests if VM configuration file exists.
4851 # We need that when migration VMs to other nodes (files already moved)
4852 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4853 sub vm_stop {
4854     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4855
4856     $force = 1 if !defined($force) && !$shutdown;
4857
4858     if ($migratedfrom){
4859         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4860         kill 15, $pid if $pid;
4861         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4862         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4863         return;
4864     }
4865
4866     PVE::QemuConfig->lock_config($vmid, sub {
4867
4868         my $pid = check_running($vmid, $nocheck);
4869         return if !$pid;
4870
4871         my $conf;
4872         if (!$nocheck) {
4873             $conf = PVE::QemuConfig->load_config($vmid);
4874             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4875             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4876                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4877                 $timeout = $opts->{down} if $opts->{down};
4878             }
4879         }
4880
4881         $timeout = 60 if !defined($timeout);
4882
4883         eval {
4884             if ($shutdown) {
4885                 if (defined($conf) && $conf->{agent}) {
4886                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4887                 } else {
4888                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4889                 }
4890             } else {
4891                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4892             }
4893         };
4894         my $err = $@;
4895
4896         if (!$err) {
4897             my $count = 0;
4898             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4899                 $count++;
4900                 sleep 1;
4901             }
4902
4903             if ($count >= $timeout) {
4904                 if ($force) {
4905                     warn "VM still running - terminating now with SIGTERM\n";
4906                     kill 15, $pid;
4907                 } else {
4908                     die "VM quit/powerdown failed - got timeout\n";
4909                 }
4910             } else {
4911                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4912                 return;
4913             }
4914         } else {
4915             if ($force) {
4916                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4917                 kill 15, $pid;
4918             } else {
4919                 die "VM quit/powerdown failed\n";
4920             }
4921         }
4922
4923         # wait again
4924         $timeout = 10;
4925
4926         my $count = 0;
4927         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4928             $count++;
4929             sleep 1;
4930         }
4931
4932         if ($count >= $timeout) {
4933             warn "VM still running - terminating now with SIGKILL\n";
4934             kill 9, $pid;
4935             sleep 1;
4936         }
4937
4938         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4939    });
4940 }
4941
4942 sub vm_suspend {
4943     my ($vmid, $skiplock) = @_;
4944
4945     PVE::QemuConfig->lock_config($vmid, sub {
4946
4947         my $conf = PVE::QemuConfig->load_config($vmid);
4948
4949         PVE::QemuConfig->check_lock($conf)
4950             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4951
4952         vm_mon_cmd($vmid, "stop");
4953     });
4954 }
4955
4956 sub vm_resume {
4957     my ($vmid, $skiplock, $nocheck) = @_;
4958
4959     PVE::QemuConfig->lock_config($vmid, sub {
4960
4961         if (!$nocheck) {
4962
4963             my $conf = PVE::QemuConfig->load_config($vmid);
4964
4965             PVE::QemuConfig->check_lock($conf)
4966                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4967
4968             vm_mon_cmd($vmid, "cont");
4969
4970         } else {
4971             vm_mon_cmd_nocheck($vmid, "cont");
4972         }
4973     });
4974 }
4975
4976 sub vm_sendkey {
4977     my ($vmid, $skiplock, $key) = @_;
4978
4979     PVE::QemuConfig->lock_config($vmid, sub {
4980
4981         my $conf = PVE::QemuConfig->load_config($vmid);
4982
4983         # there is no qmp command, so we use the human monitor command
4984         vm_human_monitor_command($vmid, "sendkey $key");
4985     });
4986 }
4987
4988 sub vm_destroy {
4989     my ($storecfg, $vmid, $skiplock) = @_;
4990
4991     PVE::QemuConfig->lock_config($vmid, sub {
4992
4993         my $conf = PVE::QemuConfig->load_config($vmid);
4994
4995         if (!check_running($vmid)) {
4996             destroy_vm($storecfg, $vmid, undef, $skiplock);
4997         } else {
4998             die "VM $vmid is running - destroy failed\n";
4999         }
5000     });
5001 }
5002
5003 # pci helpers
5004
5005 sub file_write {
5006     my ($filename, $buf) = @_;
5007
5008     my $fh = IO::File->new($filename, "w");
5009     return undef if !$fh;
5010
5011     my $res = print $fh $buf;
5012
5013     $fh->close();
5014
5015     return $res;
5016 }
5017
5018 sub pci_device_info {
5019     my ($name) = @_;
5020
5021     my $res;
5022
5023     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5024     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5025
5026     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5027     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5028
5029     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5030     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5031
5032     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5033     return undef if !defined($product) || $product !~ s/^0x//;
5034
5035     $res = {
5036         name => $name,
5037         vendor => $vendor,
5038         product => $product,
5039         domain => $domain,
5040         bus => $bus,
5041         slot => $slot,
5042         func => $func,
5043         irq => $irq,
5044         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5045     };
5046
5047     return $res;
5048 }
5049
5050 sub pci_dev_reset {
5051     my ($dev) = @_;
5052
5053     my $name = $dev->{name};
5054
5055     my $fn = "$pcisysfs/devices/$name/reset";
5056
5057     return file_write($fn, "1");
5058 }
5059
5060 sub pci_dev_bind_to_vfio {
5061     my ($dev) = @_;
5062
5063     my $name = $dev->{name};
5064
5065     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5066
5067     if (!-d $vfio_basedir) {
5068         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5069     }
5070     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5071
5072     my $testdir = "$vfio_basedir/$name";
5073     return 1 if -d $testdir;
5074
5075     my $data = "$dev->{vendor} $dev->{product}";
5076     return undef if !file_write("$vfio_basedir/new_id", $data);
5077
5078     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5079     if (!file_write($fn, $name)) {
5080         return undef if -f $fn;
5081     }
5082
5083     $fn = "$vfio_basedir/bind";
5084     if (! -d $testdir) {
5085         return undef if !file_write($fn, $name);
5086     }
5087
5088     return -d $testdir;
5089 }
5090
5091 sub pci_dev_group_bind_to_vfio {
5092     my ($pciid) = @_;
5093
5094     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5095
5096     if (!-d $vfio_basedir) {
5097         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5098     }
5099     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5100
5101     # get IOMMU group devices
5102     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5103       my @devs = grep /^0000:/, readdir($D);
5104     closedir($D);
5105
5106     foreach my $pciid (@devs) {
5107         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5108
5109         # pci bridges, switches or root ports are not supported
5110         # they have a pci_bus subdirectory so skip them
5111         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5112
5113         my $info = pci_device_info($1);
5114         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5115     }
5116
5117     return 1;
5118 }
5119
5120 # vzdump restore implementaion
5121
5122 sub tar_archive_read_firstfile {
5123     my $archive = shift;
5124
5125     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5126
5127     # try to detect archive type first
5128     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5129         die "unable to open file '$archive'\n";
5130     my $firstfile = <$fh>;
5131     kill 15, $pid;
5132     close $fh;
5133
5134     die "ERROR: archive contaions no data\n" if !$firstfile;
5135     chomp $firstfile;
5136
5137     return $firstfile;
5138 }
5139
5140 sub tar_restore_cleanup {
5141     my ($storecfg, $statfile) = @_;
5142
5143     print STDERR "starting cleanup\n";
5144
5145     if (my $fd = IO::File->new($statfile, "r")) {
5146         while (defined(my $line = <$fd>)) {
5147             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5148                 my $volid = $2;
5149                 eval {
5150                     if ($volid =~ m|^/|) {
5151                         unlink $volid || die 'unlink failed\n';
5152                     } else {
5153                         PVE::Storage::vdisk_free($storecfg, $volid);
5154                     }
5155                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5156                 };
5157                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5158             } else {
5159                 print STDERR "unable to parse line in statfile - $line";
5160             }
5161         }
5162         $fd->close();
5163     }
5164 }
5165
5166 sub restore_archive {
5167     my ($archive, $vmid, $user, $opts) = @_;
5168
5169     my $format = $opts->{format};
5170     my $comp;
5171
5172     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5173         $format = 'tar' if !$format;
5174         $comp = 'gzip';
5175     } elsif ($archive =~ m/\.tar$/) {
5176         $format = 'tar' if !$format;
5177     } elsif ($archive =~ m/.tar.lzo$/) {
5178         $format = 'tar' if !$format;
5179         $comp = 'lzop';
5180     } elsif ($archive =~ m/\.vma$/) {
5181         $format = 'vma' if !$format;
5182     } elsif ($archive =~ m/\.vma\.gz$/) {
5183         $format = 'vma' if !$format;
5184         $comp = 'gzip';
5185     } elsif ($archive =~ m/\.vma\.lzo$/) {
5186         $format = 'vma' if !$format;
5187         $comp = 'lzop';
5188     } else {
5189         $format = 'vma' if !$format; # default
5190     }
5191
5192     # try to detect archive format
5193     if ($format eq 'tar') {
5194         return restore_tar_archive($archive, $vmid, $user, $opts);
5195     } else {
5196         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5197     }
5198 }
5199
5200 sub restore_update_config_line {
5201     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5202
5203     return if $line =~ m/^\#qmdump\#/;
5204     return if $line =~ m/^\#vzdump\#/;
5205     return if $line =~ m/^lock:/;
5206     return if $line =~ m/^unused\d+:/;
5207     return if $line =~ m/^parent:/;
5208     return if $line =~ m/^template:/; # restored VM is never a template
5209
5210     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5211     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5212         # try to convert old 1.X settings
5213         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5214         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5215             my ($model, $macaddr) = split(/\=/, $devconfig);
5216             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5217             my $net = {
5218                 model => $model,
5219                 bridge => "vmbr$ind",
5220                 macaddr => $macaddr,
5221             };
5222             my $netstr = print_net($net);
5223
5224             print $outfd "net$cookie->{netcount}: $netstr\n";
5225             $cookie->{netcount}++;
5226         }
5227     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5228         my ($id, $netstr) = ($1, $2);
5229         my $net = parse_net($netstr);
5230         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5231         $netstr = print_net($net);
5232         print $outfd "$id: $netstr\n";
5233     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5234         my $virtdev = $1;
5235         my $value = $3;
5236         my $di = parse_drive($virtdev, $value);
5237         if (defined($di->{backup}) && !$di->{backup}) {
5238             print $outfd "#$line";
5239         } elsif ($map->{$virtdev}) {
5240             delete $di->{format}; # format can change on restore
5241             $di->{file} = $map->{$virtdev};
5242             $value = print_drive($vmid, $di);
5243             print $outfd "$virtdev: $value\n";
5244         } else {
5245             print $outfd $line;
5246         }
5247     } else {
5248         print $outfd $line;
5249     }
5250 }
5251
5252 sub scan_volids {
5253     my ($cfg, $vmid) = @_;
5254
5255     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5256
5257     my $volid_hash = {};
5258     foreach my $storeid (keys %$info) {
5259         foreach my $item (@{$info->{$storeid}}) {
5260             next if !($item->{volid} && $item->{size});
5261             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5262             $volid_hash->{$item->{volid}} = $item;
5263         }
5264     }
5265
5266     return $volid_hash;
5267 }
5268
5269 sub is_volume_in_use {
5270     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5271
5272     my $path = PVE::Storage::path($storecfg, $volid);
5273
5274     my $scan_config = sub {
5275         my ($cref, $snapname) = @_;
5276
5277         foreach my $key (keys %$cref) {
5278             my $value = $cref->{$key};
5279             if (is_valid_drivename($key)) {
5280                 next if $skip_drive && $key eq $skip_drive;
5281                 my $drive = parse_drive($key, $value);
5282                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5283                 return 1 if $volid eq $drive->{file};
5284                 if ($drive->{file} =~ m!^/!) {
5285                     return 1 if $drive->{file} eq $path;
5286                 } else {
5287                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5288                     next if !$storeid;
5289                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5290                     next if !$scfg;
5291                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5292                 }
5293             }
5294         }
5295
5296         return 0;
5297     };
5298
5299     return 1 if &$scan_config($conf);
5300
5301     undef $skip_drive;
5302
5303     foreach my $snapname (keys %{$conf->{snapshots}}) {
5304         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5305     }
5306
5307     return 0;
5308 }
5309
5310 sub update_disksize {
5311     my ($vmid, $conf, $volid_hash) = @_;
5312
5313     my $changes;
5314
5315     my $used = {};
5316
5317     # Note: it is allowed to define multiple storages with same path (alias), so
5318     # we need to check both 'volid' and real 'path' (two different volid can point
5319     # to the same path).
5320
5321     my $usedpath = {};
5322
5323     # update size info
5324     foreach my $opt (keys %$conf) {
5325         if (is_valid_drivename($opt)) {
5326             my $drive = parse_drive($opt, $conf->{$opt});
5327             my $volid = $drive->{file};
5328             next if !$volid;
5329
5330             $used->{$volid} = 1;
5331             if ($volid_hash->{$volid} &&
5332                 (my $path = $volid_hash->{$volid}->{path})) {
5333                 $usedpath->{$path} = 1;
5334             }
5335
5336             next if drive_is_cdrom($drive);
5337             next if !$volid_hash->{$volid};
5338
5339             $drive->{size} = $volid_hash->{$volid}->{size};
5340             my $new = print_drive($vmid, $drive);
5341             if ($new ne $conf->{$opt}) {
5342                 $changes = 1;
5343                 $conf->{$opt} = $new;
5344             }
5345         }
5346     }
5347
5348     # remove 'unusedX' entry if volume is used
5349     foreach my $opt (keys %$conf) {
5350         next if $opt !~ m/^unused\d+$/;
5351         my $volid = $conf->{$opt};
5352         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5353         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5354             $changes = 1;
5355             delete $conf->{$opt};
5356         }
5357     }
5358
5359     foreach my $volid (sort keys %$volid_hash) {
5360         next if $volid =~ m/vm-$vmid-state-/;
5361         next if $used->{$volid};
5362         my $path = $volid_hash->{$volid}->{path};
5363         next if !$path; # just to be sure
5364         next if $usedpath->{$path};
5365         $changes = 1;
5366         PVE::QemuConfig->add_unused_volume($conf, $volid);
5367         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5368     }
5369
5370     return $changes;
5371 }
5372
5373 sub rescan {
5374     my ($vmid, $nolock) = @_;
5375
5376     my $cfg = PVE::Storage::config();
5377
5378     my $volid_hash = scan_volids($cfg, $vmid);
5379
5380     my $updatefn =  sub {
5381         my ($vmid) = @_;
5382
5383         my $conf = PVE::QemuConfig->load_config($vmid);
5384
5385         PVE::QemuConfig->check_lock($conf);
5386
5387         my $vm_volids = {};
5388         foreach my $volid (keys %$volid_hash) {
5389             my $info = $volid_hash->{$volid};
5390             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5391         }
5392
5393         my $changes = update_disksize($vmid, $conf, $vm_volids);
5394
5395         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5396     };
5397
5398     if (defined($vmid)) {
5399         if ($nolock) {
5400             &$updatefn($vmid);
5401         } else {
5402             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5403         }
5404     } else {
5405         my $vmlist = config_list();
5406         foreach my $vmid (keys %$vmlist) {
5407             if ($nolock) {
5408                 &$updatefn($vmid);
5409             } else {
5410                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5411             }
5412         }
5413     }
5414 }
5415
5416 sub restore_vma_archive {
5417     my ($archive, $vmid, $user, $opts, $comp) = @_;
5418
5419     my $input = $archive eq '-' ? "<&STDIN" : undef;
5420     my $readfrom = $archive;
5421
5422     my $uncomp = '';
5423     if ($comp) {
5424         $readfrom = '-';
5425         my $qarchive = PVE::Tools::shellquote($archive);
5426         if ($comp eq 'gzip') {
5427             $uncomp = "zcat $qarchive|";
5428         } elsif ($comp eq 'lzop') {
5429             $uncomp = "lzop -d -c $qarchive|";
5430         } else {
5431             die "unknown compression method '$comp'\n";
5432         }
5433
5434     }
5435
5436     my $tmpdir = "/var/tmp/vzdumptmp$$";
5437     rmtree $tmpdir;
5438
5439     # disable interrupts (always do cleanups)
5440     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5441         warn "got interrupt - ignored\n";
5442     };
5443
5444     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5445     POSIX::mkfifo($mapfifo, 0600);
5446     my $fifofh;
5447
5448     my $openfifo = sub {
5449         open($fifofh, '>', $mapfifo) || die $!;
5450     };
5451
5452     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5453
5454     my $oldtimeout;
5455     my $timeout = 5;
5456
5457     my $devinfo = {};
5458
5459     my $rpcenv = PVE::RPCEnvironment::get();
5460
5461     my $conffile = PVE::QemuConfig->config_file($vmid);
5462     my $tmpfn = "$conffile.$$.tmp";
5463
5464     # Note: $oldconf is undef if VM does not exists
5465     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5466     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5467
5468     my $print_devmap = sub {
5469         my $virtdev_hash = {};
5470
5471         my $cfgfn = "$tmpdir/qemu-server.conf";
5472
5473         # we can read the config - that is already extracted
5474         my $fh = IO::File->new($cfgfn, "r") ||
5475             "unable to read qemu-server.conf - $!\n";
5476
5477         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5478         if (-f $fwcfgfn) {
5479             my $pve_firewall_dir = '/etc/pve/firewall';
5480             mkdir $pve_firewall_dir; # make sure the dir exists
5481             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5482         }
5483
5484         while (defined(my $line = <$fh>)) {
5485             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5486                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5487                 die "archive does not contain data for drive '$virtdev'\n"
5488                     if !$devinfo->{$devname};
5489                 if (defined($opts->{storage})) {
5490                     $storeid = $opts->{storage} || 'local';
5491                 } elsif (!$storeid) {
5492                     $storeid = 'local';
5493                 }
5494                 $format = 'raw' if !$format;
5495                 $devinfo->{$devname}->{devname} = $devname;
5496                 $devinfo->{$devname}->{virtdev} = $virtdev;
5497                 $devinfo->{$devname}->{format} = $format;
5498                 $devinfo->{$devname}->{storeid} = $storeid;
5499
5500                 # check permission on storage
5501                 my $pool = $opts->{pool}; # todo: do we need that?
5502                 if ($user ne 'root@pam') {
5503                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5504                 }
5505
5506                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5507             }
5508         }
5509
5510         foreach my $devname (keys %$devinfo) {
5511             die "found no device mapping information for device '$devname'\n"
5512                 if !$devinfo->{$devname}->{virtdev};
5513         }
5514
5515         my $cfg = PVE::Storage::config();
5516
5517         # create empty/temp config
5518         if ($oldconf) {
5519             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5520             foreach_drive($oldconf, sub {
5521                 my ($ds, $drive) = @_;
5522
5523                 return if drive_is_cdrom($drive);
5524
5525                 my $volid = $drive->{file};
5526
5527                 return if !$volid || $volid =~ m|^/|;
5528
5529                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5530                 return if !$path || !$owner || ($owner != $vmid);
5531
5532                 # Note: only delete disk we want to restore
5533                 # other volumes will become unused
5534                 if ($virtdev_hash->{$ds}) {
5535                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5536                     if (my $err = $@) {
5537                         warn $err;
5538                     }
5539                 }
5540             });
5541
5542             # delete vmstate files
5543             # since after the restore we have no snapshots anymore
5544             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5545                 my $snap = $oldconf->{snapshots}->{$snapname};
5546                 if ($snap->{vmstate}) {
5547                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5548                     if (my $err = $@) {
5549                         warn $err;
5550                     }
5551                 }
5552             }
5553         }
5554
5555         my $map = {};
5556         foreach my $virtdev (sort keys %$virtdev_hash) {
5557             my $d = $virtdev_hash->{$virtdev};
5558             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5559             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5560
5561             # test if requested format is supported
5562             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5563             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5564             $d->{format} = $defFormat if !$supported;
5565
5566             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5567                                                   $d->{format}, undef, $alloc_size);
5568             print STDERR "new volume ID is '$volid'\n";
5569             $d->{volid} = $volid;
5570             my $path = PVE::Storage::path($cfg, $volid);
5571
5572             PVE::Storage::activate_volumes($cfg,[$volid]);
5573
5574             my $write_zeros = 1;
5575             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5576                 $write_zeros = 0;
5577             }
5578
5579             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5580
5581             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5582             $map->{$virtdev} = $volid;
5583         }
5584
5585         $fh->seek(0, 0) || die "seek failed - $!\n";
5586
5587         my $outfd = new IO::File ($tmpfn, "w") ||
5588             die "unable to write config for VM $vmid\n";
5589
5590         my $cookie = { netcount => 0 };
5591         while (defined(my $line = <$fh>)) {
5592             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5593         }
5594
5595         $fh->close();
5596         $outfd->close();
5597     };
5598
5599     eval {
5600         # enable interrupts
5601         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5602             die "interrupted by signal\n";
5603         };
5604         local $SIG{ALRM} = sub { die "got timeout\n"; };
5605
5606         $oldtimeout = alarm($timeout);
5607
5608         my $parser = sub {
5609             my $line = shift;
5610
5611             print "$line\n";
5612
5613             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5614                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5615                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5616             } elsif ($line =~ m/^CTIME: /) {
5617                 # we correctly received the vma config, so we can disable
5618                 # the timeout now for disk allocation (set to 10 minutes, so
5619                 # that we always timeout if something goes wrong)
5620                 alarm(600);
5621                 &$print_devmap();
5622                 print $fifofh "done\n";
5623                 my $tmp = $oldtimeout || 0;
5624                 $oldtimeout = undef;
5625                 alarm($tmp);
5626                 close($fifofh);
5627             }
5628         };
5629
5630         print "restore vma archive: $cmd\n";
5631         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5632     };
5633     my $err = $@;
5634
5635     alarm($oldtimeout) if $oldtimeout;
5636
5637     my $vollist = [];
5638     foreach my $devname (keys %$devinfo) {
5639         my $volid = $devinfo->{$devname}->{volid};
5640         push @$vollist, $volid if $volid;
5641     }
5642
5643     my $cfg = PVE::Storage::config();
5644     PVE::Storage::deactivate_volumes($cfg, $vollist);
5645
5646     unlink $mapfifo;
5647
5648     if ($err) {
5649         rmtree $tmpdir;
5650         unlink $tmpfn;
5651
5652         foreach my $devname (keys %$devinfo) {
5653             my $volid = $devinfo->{$devname}->{volid};
5654             next if !$volid;
5655             eval {
5656                 if ($volid =~ m|^/|) {
5657                     unlink $volid || die 'unlink failed\n';
5658                 } else {
5659                     PVE::Storage::vdisk_free($cfg, $volid);
5660                 }
5661                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5662             };
5663             print STDERR "unable to cleanup '$volid' - $@" if $@;
5664         }
5665         die $err;
5666     }
5667
5668     rmtree $tmpdir;
5669
5670     rename($tmpfn, $conffile) ||
5671         die "unable to commit configuration file '$conffile'\n";
5672
5673     PVE::Cluster::cfs_update(); # make sure we read new file
5674
5675     eval { rescan($vmid, 1); };
5676     warn $@ if $@;
5677 }
5678
5679 sub restore_tar_archive {
5680     my ($archive, $vmid, $user, $opts) = @_;
5681
5682     if ($archive ne '-') {
5683         my $firstfile = tar_archive_read_firstfile($archive);
5684         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5685             if $firstfile ne 'qemu-server.conf';
5686     }
5687
5688     my $storecfg = PVE::Storage::config();
5689
5690     # destroy existing data - keep empty config
5691     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5692     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5693
5694     my $tocmd = "/usr/lib/qemu-server/qmextract";
5695
5696     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5697     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5698     $tocmd .= ' --prealloc' if $opts->{prealloc};
5699     $tocmd .= ' --info' if $opts->{info};
5700
5701     # tar option "xf" does not autodetect compression when read from STDIN,
5702     # so we pipe to zcat
5703     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5704         PVE::Tools::shellquote("--to-command=$tocmd");
5705
5706     my $tmpdir = "/var/tmp/vzdumptmp$$";
5707     mkpath $tmpdir;
5708
5709     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5710     local $ENV{VZDUMP_VMID} = $vmid;
5711     local $ENV{VZDUMP_USER} = $user;
5712
5713     my $conffile = PVE::QemuConfig->config_file($vmid);
5714     my $tmpfn = "$conffile.$$.tmp";
5715
5716     # disable interrupts (always do cleanups)
5717     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5718         print STDERR "got interrupt - ignored\n";
5719     };
5720
5721     eval {
5722         # enable interrupts
5723         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5724             die "interrupted by signal\n";
5725         };
5726
5727         if ($archive eq '-') {
5728             print "extracting archive from STDIN\n";
5729             run_command($cmd, input => "<&STDIN");
5730         } else {
5731             print "extracting archive '$archive'\n";
5732             run_command($cmd);
5733         }
5734
5735         return if $opts->{info};
5736
5737         # read new mapping
5738         my $map = {};
5739         my $statfile = "$tmpdir/qmrestore.stat";
5740         if (my $fd = IO::File->new($statfile, "r")) {
5741             while (defined (my $line = <$fd>)) {
5742                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5743                     $map->{$1} = $2 if $1;
5744                 } else {
5745                     print STDERR "unable to parse line in statfile - $line\n";
5746                 }
5747             }
5748             $fd->close();
5749         }
5750
5751         my $confsrc = "$tmpdir/qemu-server.conf";
5752
5753         my $srcfd = new IO::File($confsrc, "r") ||
5754             die "unable to open file '$confsrc'\n";
5755
5756         my $outfd = new IO::File ($tmpfn, "w") ||
5757             die "unable to write config for VM $vmid\n";
5758
5759         my $cookie = { netcount => 0 };
5760         while (defined (my $line = <$srcfd>)) {
5761             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5762         }
5763
5764         $srcfd->close();
5765         $outfd->close();
5766     };
5767     my $err = $@;
5768
5769     if ($err) {
5770
5771         unlink $tmpfn;
5772
5773         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5774
5775         die $err;
5776     }
5777
5778     rmtree $tmpdir;
5779
5780     rename $tmpfn, $conffile ||
5781         die "unable to commit configuration file '$conffile'\n";
5782
5783     PVE::Cluster::cfs_update(); # make sure we read new file
5784
5785     eval { rescan($vmid, 1); };
5786     warn $@ if $@;
5787 };
5788
5789 sub foreach_writable_storage {
5790     my ($conf, $func) = @_;
5791
5792     my $sidhash = {};
5793
5794     foreach my $ds (keys %$conf) {
5795         next if !is_valid_drivename($ds);
5796
5797         my $drive = parse_drive($ds, $conf->{$ds});
5798         next if !$drive;
5799         next if drive_is_cdrom($drive);
5800
5801         my $volid = $drive->{file};
5802
5803         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5804         $sidhash->{$sid} = $sid if $sid;
5805     }
5806
5807     foreach my $sid (sort keys %$sidhash) {
5808         &$func($sid);
5809     }
5810 }
5811
5812 sub do_snapshots_with_qemu {
5813     my ($storecfg, $volid) = @_;
5814
5815     my $storage_name = PVE::Storage::parse_volume_id($volid);
5816
5817     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5818         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5819         return 1;
5820     }
5821
5822     if ($volid =~ m/\.(qcow2|qed)$/){
5823         return 1;
5824     }
5825
5826     return undef;
5827 }
5828
5829 sub qga_check_running {
5830     my ($vmid) = @_;
5831
5832     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5833     if ($@) {
5834         warn "Qemu Guest Agent are not running - $@";
5835         return 0;
5836     }
5837     return 1;
5838 }
5839
5840 sub template_create {
5841     my ($vmid, $conf, $disk) = @_;
5842
5843     my $storecfg = PVE::Storage::config();
5844
5845     foreach_drive($conf, sub {
5846         my ($ds, $drive) = @_;
5847
5848         return if drive_is_cdrom($drive);
5849         return if $disk && $ds ne $disk;
5850
5851         my $volid = $drive->{file};
5852         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5853
5854         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5855         $drive->{file} = $voliddst;
5856         $conf->{$ds} = print_drive($vmid, $drive);
5857         PVE::QemuConfig->write_config($vmid, $conf);
5858     });
5859 }
5860
5861 sub qemu_img_convert {
5862     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5863
5864     my $storecfg = PVE::Storage::config();
5865     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5866     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5867
5868     if ($src_storeid && $dst_storeid) {
5869
5870         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5871
5872         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5873         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5874
5875         my $src_format = qemu_img_format($src_scfg, $src_volname);
5876         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5877
5878         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5879         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5880
5881         my $cmd = [];
5882         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5883         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5884         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5885         if ($is_zero_initialized) {
5886             push @$cmd, "zeroinit:$dst_path";
5887         } else {
5888             push @$cmd, $dst_path;
5889         }
5890
5891         my $parser = sub {
5892             my $line = shift;
5893             if($line =~ m/\((\S+)\/100\%\)/){
5894                 my $percent = $1;
5895                 my $transferred = int($size * $percent / 100);
5896                 my $remaining = $size - $transferred;
5897
5898                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5899             }
5900
5901         };
5902
5903         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5904         my $err = $@;
5905         die "copy failed: $err" if $err;
5906     }
5907 }
5908
5909 sub qemu_img_format {
5910     my ($scfg, $volname) = @_;
5911
5912     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5913         return $1;
5914     } else {
5915         return "raw";
5916     }
5917 }
5918
5919 sub qemu_drive_mirror {
5920     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
5921
5922     $jobs = {} if !$jobs;
5923
5924     my $qemu_target;
5925     my $format;
5926     $jobs->{"drive-$drive"} = {};
5927
5928     if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
5929         my $server = $1;
5930         my $port = $2;
5931         my $exportname = $3;
5932
5933         $format = "nbd";
5934         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
5935         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
5936         my $cmd = ['socat', '-T30', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=5"];
5937
5938         my $pid = fork();
5939         if (!defined($pid)) {
5940             die "forking socat tunnel failed\n";
5941         } elsif ($pid == 0) {
5942             exec(@$cmd);
5943             warn "exec failed: $!\n";
5944             POSIX::_exit(-1);
5945         }
5946         $jobs->{"drive-$drive"}->{pid} = $pid;
5947
5948         my $timeout = 0;
5949         while (!-S $unixsocket) {
5950             die "nbd connection helper timed out\n"
5951                 if $timeout++ > 5;
5952             sleep 1;
5953         }
5954     } else {
5955         my $storecfg = PVE::Storage::config();
5956         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5957
5958         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5959
5960         $format = qemu_img_format($dst_scfg, $dst_volname);
5961
5962         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5963
5964         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5965     }
5966
5967     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5968     $opts->{format} = $format if $format;
5969
5970     print "drive mirror is starting for drive-$drive\n";
5971
5972     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
5973
5974     if (my $err = $@) {
5975         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
5976         die "mirroring error: $err";
5977     }
5978
5979     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
5980 }
5981
5982 sub qemu_drive_mirror_monitor {
5983     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
5984
5985     eval {
5986         my $err_complete = 0;
5987
5988         while (1) {
5989             die "storage migration timed out\n" if $err_complete > 300;
5990
5991             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5992
5993             my $running_mirror_jobs = {};
5994             foreach my $stat (@$stats) {
5995                 next if $stat->{type} ne 'mirror';
5996                 $running_mirror_jobs->{$stat->{device}} = $stat;
5997             }
5998
5999             my $readycounter = 0;
6000
6001             foreach my $job (keys %$jobs) {
6002
6003                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6004                     print "$job : finished\n";
6005                     delete $jobs->{$job};
6006                     next;
6007                 }
6008
6009                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6010
6011                 my $busy = $running_mirror_jobs->{$job}->{busy};
6012                 my $ready = $running_mirror_jobs->{$job}->{ready};
6013                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6014                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6015                     my $remaining = $total - $transferred;
6016                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6017
6018                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6019                 }
6020
6021                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6022             }
6023
6024             last if scalar(keys %$jobs) == 0;
6025
6026             if ($readycounter == scalar(keys %$jobs)) {
6027                 print "all mirroring jobs are ready \n";
6028                 last if $skipcomplete; #do the complete later
6029
6030                 if ($vmiddst && $vmiddst != $vmid) {
6031                     if ($qga) {
6032                         print "freeze filesystem\n";
6033                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6034                     } else {
6035                         print "suspend vm\n";
6036                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6037                     }
6038
6039                     # if we clone a disk for a new target vm, we don't switch the disk
6040                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6041
6042                     if ($qga) {
6043                         print "unfreeze filesystem\n";
6044                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6045                     } else {
6046                         print "resume vm\n";
6047                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6048                     }
6049
6050                     last;
6051                 } else {
6052
6053                     foreach my $job (keys %$jobs) {
6054                         # try to switch the disk if source and destination are on the same guest
6055                         print "$job: Completing block job...\n";
6056
6057                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6058                         if ($@ =~ m/cannot be completed/) {
6059                             print "$job: Block job cannot be completed, try again.\n";
6060                             $err_complete++;
6061                         }else {
6062                             print "$job: Completed successfully.\n";
6063                             $jobs->{$job}->{complete} = 1;
6064                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6065                         }
6066                     }
6067                 }
6068             }
6069             sleep 1;
6070         }
6071     };
6072     my $err = $@;
6073
6074     if ($err) {
6075         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6076         die "mirroring error: $err";
6077     }
6078
6079 }
6080
6081 sub qemu_blockjobs_cancel {
6082     my ($vmid, $jobs) = @_;
6083
6084     foreach my $job (keys %$jobs) {
6085         print "$job: Cancelling block job\n";
6086         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6087         $jobs->{$job}->{cancel} = 1;
6088     }
6089
6090     while (1) {
6091         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6092
6093         my $running_jobs = {};
6094         foreach my $stat (@$stats) {
6095             $running_jobs->{$stat->{device}} = $stat;
6096         }
6097
6098         foreach my $job (keys %$jobs) {
6099
6100             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6101                 print "$job: Done.\n";
6102                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6103                 delete $jobs->{$job};
6104             }
6105         }
6106
6107         last if scalar(keys %$jobs) == 0;
6108
6109         sleep 1;
6110     }
6111 }
6112
6113 sub qemu_blockjobs_finish_tunnel {
6114    my ($vmid, $job, $cpid) = @_;
6115
6116    return if !$cpid;
6117
6118    for (my $i = 1; $i < 20; $i++) {
6119         my $waitpid = waitpid($cpid, WNOHANG);
6120         last if (defined($waitpid) && ($waitpid == $cpid));
6121
6122         if ($i == 10) {
6123             kill(15, $cpid);
6124         } elsif ($i >= 15) {
6125             kill(9, $cpid);
6126         }
6127         sleep (1);
6128     }
6129     unlink "/run/qemu-server/$vmid.mirror-$job";
6130 }
6131
6132 sub clone_disk {
6133     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6134         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6135
6136     my $newvolid;
6137
6138     if (!$full) {
6139         print "create linked clone of drive $drivename ($drive->{file})\n";
6140         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6141         push @$newvollist, $newvolid;
6142     } else {
6143
6144         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6145         $storeid = $storage if $storage;
6146
6147         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6148         if (!$format) {
6149             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6150             $format = qemu_img_format($scfg, $volname);
6151         }
6152
6153         # test if requested format is supported - else use default
6154         my $supported = grep { $_ eq $format } @$validFormats;
6155         $format = $defFormat if !$supported;
6156
6157         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6158
6159         print "create full clone of drive $drivename ($drive->{file})\n";
6160         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6161         push @$newvollist, $newvolid;
6162
6163         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6164
6165         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6166         if (!$running || $snapname) {
6167             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6168         } else {
6169
6170             my $kvmver = get_running_qemu_version ($vmid);
6171             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6172                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6173                     if $drive->{iothread};
6174             }
6175
6176             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6177         }
6178     }
6179
6180     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6181
6182     my $disk = $drive;
6183     $disk->{format} = undef;
6184     $disk->{file} = $newvolid;
6185     $disk->{size} = $size;
6186
6187     return $disk;
6188 }
6189
6190 # this only works if VM is running
6191 sub get_current_qemu_machine {
6192     my ($vmid) = @_;
6193
6194     my $cmd = { execute => 'query-machines', arguments => {} };
6195     my $res = vm_qmp_command($vmid, $cmd);
6196
6197     my ($current, $default);
6198     foreach my $e (@$res) {
6199         $default = $e->{name} if $e->{'is-default'};
6200         $current = $e->{name} if $e->{'is-current'};
6201     }
6202
6203     # fallback to the default machine if current is not supported by qemu
6204     return $current || $default || 'pc';
6205 }
6206
6207 sub get_running_qemu_version {
6208     my ($vmid) = @_;
6209     my $cmd = { execute => 'query-version', arguments => {} };
6210     my $res = vm_qmp_command($vmid, $cmd);
6211     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6212 }
6213
6214 sub qemu_machine_feature_enabled {
6215     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6216
6217     my $current_major;
6218     my $current_minor;
6219
6220     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6221
6222         $current_major = $3;
6223         $current_minor = $4;
6224
6225     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6226
6227         $current_major = $1;
6228         $current_minor = $2;
6229     }
6230
6231     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6232
6233
6234 }
6235
6236 sub qemu_machine_pxe {
6237     my ($vmid, $conf, $machine) = @_;
6238
6239     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6240
6241     foreach my $opt (keys %$conf) {
6242         next if $opt !~ m/^net(\d+)$/;
6243         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6244         next if !$net;
6245         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6246         return $machine.".pxe" if $romfile =~ m/pxe/;
6247         last;
6248     }
6249
6250     return $machine;
6251 }
6252
6253 sub qemu_use_old_bios_files {
6254     my ($machine_type) = @_;
6255
6256     return if !$machine_type;
6257
6258     my $use_old_bios_files = undef;
6259
6260     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6261         $machine_type = $1;
6262         $use_old_bios_files = 1;
6263     } else {
6264         my $kvmver = kvm_user_version();
6265         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6266         # load new efi bios files on migration. So this hack is required to allow
6267         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6268         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6269         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6270     }
6271
6272     return ($use_old_bios_files, $machine_type);
6273 }
6274
6275 sub lspci {
6276
6277     my $devices = {};
6278
6279     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6280             my (undef, $id, $function) = @_;
6281             my $res = { id => $id, function => $function};
6282             push @{$devices->{$id}}, $res;
6283     });
6284
6285     return $devices;
6286 }
6287
6288 sub vm_iothreads_list {
6289     my ($vmid) = @_;
6290
6291     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6292
6293     my $iothreads = {};
6294     foreach my $iothread (@$res) {
6295         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6296     }
6297
6298     return $iothreads;
6299 }
6300
6301 sub scsihw_infos {
6302     my ($conf, $drive) = @_;
6303
6304     my $maxdev = 0;
6305
6306     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6307         $maxdev = 7;
6308     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6309         $maxdev = 1;
6310     } else {
6311         $maxdev = 256;
6312     }
6313
6314     my $controller = int($drive->{index} / $maxdev);
6315     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6316
6317     return ($maxdev, $controller, $controller_prefix);
6318 }
6319
6320 sub add_hyperv_enlighments {
6321     my ($cpuFlags, $winversion, $machine_type, $kvmver, $nokvm, $bios, $gpu_passthrough) = @_;
6322
6323     return if $nokvm;
6324     return if $winversion < 6;
6325     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6326
6327     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6328
6329     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6330         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6331         push @$cpuFlags , 'hv_vapic';
6332         push @$cpuFlags , 'hv_time';
6333     } else {
6334         push @$cpuFlags , 'hv_spinlocks=0xffff';
6335     }
6336
6337     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6338         push @$cpuFlags , 'hv_reset';
6339         push @$cpuFlags , 'hv_vpindex';
6340         push @$cpuFlags , 'hv_runtime';
6341     }
6342
6343     if ($winversion >= 7) {
6344         push @$cpuFlags , 'hv_relaxed';
6345     }
6346 }
6347
6348 sub windows_version {
6349     my ($ostype) = @_;
6350
6351     return 0 if !$ostype;
6352
6353     my $winversion = 0;
6354
6355     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6356         $winversion = 5;
6357     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6358         $winversion = 6;
6359     } elsif ($ostype =~ m/^win(\d+)$/) {
6360         $winversion = $1;
6361     }
6362
6363     return $winversion;
6364 }
6365
6366 # bash completion helper
6367
6368 sub complete_backup_archives {
6369     my ($cmdname, $pname, $cvalue) = @_;
6370
6371     my $cfg = PVE::Storage::config();
6372
6373     my $storeid;
6374
6375     if ($cvalue =~ m/^([^:]+):/) {
6376         $storeid = $1;
6377     }
6378
6379     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6380
6381     my $res = [];
6382     foreach my $id (keys %$data) {
6383         foreach my $item (@{$data->{$id}}) {
6384             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6385             push @$res, $item->{volid} if defined($item->{volid});
6386         }
6387     }
6388
6389     return $res;
6390 }
6391
6392 my $complete_vmid_full = sub {
6393     my ($running) = @_;
6394
6395     my $idlist = vmstatus();
6396
6397     my $res = [];
6398
6399     foreach my $id (keys %$idlist) {
6400         my $d = $idlist->{$id};
6401         if (defined($running)) {
6402             next if $d->{template};
6403             next if $running && $d->{status} ne 'running';
6404             next if !$running && $d->{status} eq 'running';
6405         }
6406         push @$res, $id;
6407
6408     }
6409     return $res;
6410 };
6411
6412 sub complete_vmid {
6413     return &$complete_vmid_full();
6414 }
6415
6416 sub complete_vmid_stopped {
6417     return &$complete_vmid_full(0);
6418 }
6419
6420 sub complete_vmid_running {
6421     return &$complete_vmid_full(1);
6422 }
6423
6424 sub complete_storage {
6425
6426     my $cfg = PVE::Storage::config();
6427     my $ids = $cfg->{ids};
6428
6429     my $res = [];
6430     foreach my $sid (keys %$ids) {
6431         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6432         next if !$ids->{$sid}->{content}->{images};
6433         push @$res, $sid;
6434     }
6435
6436     return $res;
6437 }
6438
6439 sub nbd_stop {
6440     my ($vmid) = @_;
6441
6442     vm_mon_cmd($vmid, 'nbd-server-stop');
6443 }
6444
6445 1;