PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42 my $OVMF_IMG = '/usr/share/kvm/OVMF-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 # Note about locking: we use flock on the config file protect
  49 # against concurent actions.
  50 # Aditionaly, we have a 'lock' setting in the config file. This
  51 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  52 # allowed when such lock is set. But you can ignore this kind of
  53 # lock with the --skiplock flag.
  54
  55 cfs_register_file('/qemu-server/',
  56                   \&parse_vm_config,
  57                   \&write_vm_config);
  58
  59 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  60     description => "Some command save/restore state from this location.",
  61     type => 'string',
  62     maxLength => 128,
  63     optional => 1,
  64 });
  65
  66 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  67     description => "The name of the snapshot.",
  68     type => 'string', format => 'pve-configid',
  69     maxLength => 40,
  70 });
  71
  72 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  73     type => 'string',
  74     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  75     description => "The drive's backing file's data format.",
  76     optional => 1,
  77 });
  78
  79 #no warnings 'redefine';
  80
  81 sub cgroups_write {
  82    my ($controller, $vmid, $option, $value) = @_;
  83
  84    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  85    PVE::ProcFSTools::write_proc_entry($path, $value);
  86
  87 }
  88
  89 my $nodename = PVE::INotify::nodename();
  90
  91 mkdir "/etc/pve/nodes/$nodename";
  92 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  93 mkdir $confdir;
  94
  95 my $var_run_tmpdir = "/var/run/qemu-server";
  96 mkdir $var_run_tmpdir;
  97
  98 my $lock_dir = "/var/lock/qemu-server";
  99 mkdir $lock_dir;
 100
 101 my $pcisysfs = "/sys/bus/pci";
 102
 103 my $cpu_vendor_list = {
 104     # Intel CPUs
 105     486 => 'GenuineIntel',
 106     pentium => 'GenuineIntel',
 107     pentium2  => 'GenuineIntel',
 108     pentium3  => 'GenuineIntel',
 109     coreduo => 'GenuineIntel',
 110     core2duo => 'GenuineIntel',
 111     Conroe  => 'GenuineIntel',
 112     Penryn  => 'GenuineIntel',
 113     Nehalem  => 'GenuineIntel',
 114     Westmere => 'GenuineIntel',
 115     SandyBridge => 'GenuineIntel',
 116     IvyBridge => 'GenuineIntel',
 117     Haswell => 'GenuineIntel',
 118     'Haswell-noTSX' => 'GenuineIntel',
 119     Broadwell => 'GenuineIntel',
 120     'Broadwell-noTSX' => 'GenuineIntel',
 121     'Skylake-Client' => 'GenuineIntel',
 122
 123     # AMD CPUs
 124     athlon => 'AuthenticAMD',
 125     phenom  => 'AuthenticAMD',
 126     Opteron_G1  => 'AuthenticAMD',
 127     Opteron_G2  => 'AuthenticAMD',
 128     Opteron_G3  => 'AuthenticAMD',
 129     Opteron_G4  => 'AuthenticAMD',
 130     Opteron_G5  => 'AuthenticAMD',
 131
 132     # generic types, use vendor from host node
 133     host => 'default',
 134     kvm32 => 'default',
 135     kvm64 => 'default',
 136     qemu32 => 'default',
 137     qemu64 => 'default',
 138 };
 139
 140 my $cpu_fmt = {
 141     cputype => {
 142         description => "Emulated CPU type.",
 143         type => 'string',
 144         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 145         default => 'kvm64',
 146         default_key => 1,
 147     },
 148     hidden => {
 149         description => "Do not identify as a KVM virtual machine.",
 150         type => 'boolean',
 151         optional => 1,
 152         default => 0
 153     },
 154 };
 155
 156 my $watchdog_fmt = {
 157     model => {
 158         default_key => 1,
 159         type => 'string',
 160         enum => [qw(i6300esb ib700)],
 161         description => "Watchdog type to emulate.",
 162         default => 'i6300esb',
 163         optional => 1,
 164     },
 165     action => {
 166         type => 'string',
 167         enum => [qw(reset shutdown poweroff pause debug none)],
 168         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 169         optional => 1,
 170     },
 171 };
 172 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 173
 174 my $confdesc = {
 175     onboot => {
 176         optional => 1,
 177         type => 'boolean',
 178         description => "Specifies whether a VM will be started during system bootup.",
 179         default => 0,
 180     },
 181     autostart => {
 182         optional => 1,
 183         type => 'boolean',
 184         description => "Automatic restart after crash (currently ignored).",
 185         default => 0,
 186     },
 187     hotplug => {
 188         optional => 1,
 189         type => 'string', format => 'pve-hotplug-features',
 190         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 191         default => 'network,disk,usb',
 192     },
 193     reboot => {
 194         optional => 1,
 195         type => 'boolean',
 196         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 197         default => 1,
 198     },
 199     lock => {
 200         optional => 1,
 201         type => 'string',
 202         description => "Lock/unlock the VM.",
 203         enum => [qw(migrate backup snapshot rollback)],
 204     },
 205     cpulimit => {
 206         optional => 1,
 207         type => 'number',
 208         description => "Limit of CPU usage.",
 209         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 210         minimum => 0,
 211         maximum => 128,
 212         default => 0,
 213     },
 214     cpuunits => {
 215         optional => 1,
 216         type => 'integer',
 217         description => "CPU weight for a VM.",
 218         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 219         minimum => 0,
 220         maximum => 500000,
 221         default => 1024,
 222     },
 223     memory => {
 224         optional => 1,
 225         type => 'integer',
 226         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 227         minimum => 16,
 228         default => 512,
 229     },
 230     balloon => {
 231         optional => 1,
 232         type => 'integer',
 233         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 234         minimum => 0,
 235     },
 236     shares => {
 237         optional => 1,
 238         type => 'integer',
 239         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 240         minimum => 0,
 241         maximum => 50000,
 242         default => 1000,
 243     },
 244     keyboard => {
 245         optional => 1,
 246         type => 'string',
 247         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 248         enum => PVE::Tools::kvmkeymaplist(),
 249         default => 'en-us',
 250     },
 251     name => {
 252         optional => 1,
 253         type => 'string', format => 'dns-name',
 254         description => "Set a name for the VM. Only used on the configuration web interface.",
 255     },
 256     scsihw => {
 257         optional => 1,
 258         type => 'string',
 259         description => "SCSI controller model",
 260         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 261         default => 'lsi',
 262     },
 263     description => {
 264         optional => 1,
 265         type => 'string',
 266         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 267     },
 268     ostype => {
 269         optional => 1,
 270         type => 'string',
 271         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 272         description => "Specify guest operating system.",
 273         verbose_description => <<EODESC,
 274 Specify guest operating system. This is used to enable special
 275 optimization/features for specific operating systems:
 276
 277 [horizontal]
 278 other;; unspecified OS
 279 wxp;; Microsoft Windows XP
 280 w2k;; Microsoft Windows 2000
 281 w2k3;; Microsoft Windows 2003
 282 w2k8;; Microsoft Windows 2008
 283 wvista;; Microsoft Windows Vista
 284 win7;; Microsoft Windows 7
 285 win8;; Microsoft Windows 8/2012
 286 l24;; Linux 2.4 Kernel
 287 l26;; Linux 2.6/3.X Kernel
 288 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 289 EODESC
 290     },
 291     boot => {
 292         optional => 1,
 293         type => 'string',
 294         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 295         pattern => '[acdn]{1,4}',
 296         default => 'cdn',
 297     },
 298     bootdisk => {
 299         optional => 1,
 300         type => 'string', format => 'pve-qm-bootdisk',
 301         description => "Enable booting from specified disk.",
 302         pattern => '(ide|sata|scsi|virtio)\d+',
 303     },
 304     smp => {
 305         optional => 1,
 306         type => 'integer',
 307         description => "The number of CPUs. Please use option -sockets instead.",
 308         minimum => 1,
 309         default => 1,
 310     },
 311     sockets => {
 312         optional => 1,
 313         type => 'integer',
 314         description => "The number of CPU sockets.",
 315         minimum => 1,
 316         default => 1,
 317     },
 318     cores => {
 319         optional => 1,
 320         type => 'integer',
 321         description => "The number of cores per socket.",
 322         minimum => 1,
 323         default => 1,
 324     },
 325     numa => {
 326         optional => 1,
 327         type => 'boolean',
 328         description => "Enable/disable NUMA.",
 329         default => 0,
 330     },
 331     hugepages => {
 332         optional => 1,
 333         type => 'string',
 334         description => "Enable/disable hugepages memory.",
 335         enum => [qw(any 2 1024)],
 336     },
 337     vcpus => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "Number of hotplugged vcpus.",
 341         minimum => 1,
 342         default => 0,
 343     },
 344     acpi => {
 345         optional => 1,
 346         type => 'boolean',
 347         description => "Enable/disable ACPI.",
 348         default => 1,
 349     },
 350     agent => {
 351         optional => 1,
 352         type => 'boolean',
 353         description => "Enable/disable Qemu GuestAgent.",
 354         default => 0,
 355     },
 356     kvm => {
 357         optional => 1,
 358         type => 'boolean',
 359         description => "Enable/disable KVM hardware virtualization.",
 360         default => 1,
 361     },
 362     tdf => {
 363         optional => 1,
 364         type => 'boolean',
 365         description => "Enable/disable time drift fix.",
 366         default => 0,
 367     },
 368     localtime => {
 369         optional => 1,
 370         type => 'boolean',
 371         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 372     },
 373     freeze => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 377     },
 378     vga => {
 379         optional => 1,
 380         type => 'string',
 381         description => "Select the VGA type.",
 382         verbose_description => "Select the VGA type. If you want to use high resolution" .
 383             " modes (>= 1280x1024x16) then you should use the options " .
 384             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 385             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 386             "display sever. For win* OS you can select how many independent " .
 387             "displays you want, Linux guests can add displays them self. " .
 388             "You can also run without any graphic card, using a serial device" .
 389             " as terminal.",
 390         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 391     },
 392     watchdog => {
 393         optional => 1,
 394         type => 'string', format => 'pve-qm-watchdog',
 395         description => "Create a virtual hardware watchdog device.",
 396         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 397             " (by a guest action), the watchdog must be periodically polled " .
 398             "by an agent inside the guest or else the watchdog will reset " .
 399             "the guest (or execute the respective action specified)",
 400     },
 401     startdate => {
 402         optional => 1,
 403         type => 'string',
 404         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 405         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 406         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 407         default => 'now',
 408     },
 409     startup =>  get_standard_option('pve-startup-order'),
 410     template => {
 411         optional => 1,
 412         type => 'boolean',
 413         description => "Enable/disable Template.",
 414         default => 0,
 415     },
 416     args => {
 417         optional => 1,
 418         type => 'string',
 419         description => "Arbitrary arguments passed to kvm.",
 420         verbose_description => <<EODESCR,
 421 Arbitrary arguments passed to kvm, for example:
 422
 423 args: -no-reboot -no-hpet
 424
 425 NOTE: this option is for experts only.
 426 EODESCR
 427     },
 428     tablet => {
 429         optional => 1,
 430         type => 'boolean',
 431         default => 1,
 432         description => "Enable/disable the USB tablet device.",
 433         verbose_description => "Enable/disable the USB tablet device. This device is " .
 434             "usually needed to allow absolute mouse positioning with VNC. " .
 435             "Else the mouse runs out of sync with normal VNC clients. " .
 436             "If you're running lots of console-only guests on one host, " .
 437             "you may consider disabling this to save some context switches. " .
 438             "This is turned off by default if you use spice (-vga=qxl).",
 439     },
 440     migrate_speed => {
 441         optional => 1,
 442         type => 'integer',
 443         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 444         minimum => 0,
 445         default => 0,
 446     },
 447     migrate_downtime => {
 448         optional => 1,
 449         type => 'number',
 450         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 451         minimum => 0,
 452         default => 0.1,
 453     },
 454     cdrom => {
 455         optional => 1,
 456         type => 'string', format => 'pve-qm-ide',
 457         typetext => '<volume>',
 458         description => "This is an alias for option -ide2",
 459     },
 460     cpu => {
 461         optional => 1,
 462         description => "Emulated CPU type.",
 463         type => 'string',
 464         format => $cpu_fmt,
 465     },
 466     parent => get_standard_option('pve-snapshot-name', {
 467         optional => 1,
 468         description => "Parent snapshot name. This is used internally, and should not be modified.",
 469     }),
 470     snaptime => {
 471         optional => 1,
 472         description => "Timestamp for snapshots.",
 473         type => 'integer',
 474         minimum => 0,
 475     },
 476     vmstate => {
 477         optional => 1,
 478         type => 'string', format => 'pve-volume-id',
 479         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 480     },
 481     machine => {
 482         description => "Specific the Qemu machine type.",
 483         type => 'string',
 484         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 485         maxLength => 40,
 486         optional => 1,
 487     },
 488     smbios1 => {
 489         description => "Specify SMBIOS type 1 fields.",
 490         type => 'string', format => 'pve-qm-smbios1',
 491         maxLength => 256,
 492         optional => 1,
 493     },
 494     protection => {
 495         optional => 1,
 496         type => 'boolean',
 497         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 498         default => 0,
 499     },
 500     bios => {
 501         optional => 1,
 502         type => 'string',
 503         enum => [ qw(seabios ovmf) ],
 504         description => "Select BIOS implementation.",
 505         default => 'seabios',
 506     },
 507 };
 508
 509 # what about other qemu settings ?
 510 #cpu => 'string',
 511 #machine => 'string',
 512 #fda => 'file',
 513 #fdb => 'file',
 514 #mtdblock => 'file',
 515 #sd => 'file',
 516 #pflash => 'file',
 517 #snapshot => 'bool',
 518 #bootp => 'file',
 519 ##tftp => 'dir',
 520 ##smb => 'dir',
 521 #kernel => 'file',
 522 #append => 'string',
 523 #initrd => 'file',
 524 ##soundhw => 'string',
 525
 526 while (my ($k, $v) = each %$confdesc) {
 527     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 528 }
 529
 530 my $MAX_IDE_DISKS = 4;
 531 my $MAX_SCSI_DISKS = 14;
 532 my $MAX_VIRTIO_DISKS = 16;
 533 my $MAX_SATA_DISKS = 6;
 534 my $MAX_USB_DEVICES = 5;
 535 my $MAX_NETS = 32;
 536 my $MAX_UNUSED_DISKS = 8;
 537 my $MAX_HOSTPCI_DEVICES = 4;
 538 my $MAX_SERIAL_PORTS = 4;
 539 my $MAX_PARALLEL_PORTS = 3;
 540 my $MAX_NUMA = 8;
 541
 542 my $numa_fmt = {
 543     cpus => {
 544         type => "string",
 545         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 546         description => "CPUs accessing this NUMA node.",
 547         format_description => "id[-id];...",
 548     },
 549     memory => {
 550         type => "number",
 551         description => "Amount of memory this NUMA node provides.",
 552         optional => 1,
 553     },
 554     hostnodes => {
 555         type => "string",
 556         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 557         description => "Host NUMA nodes to use.",
 558         format_description => "id[-id];...",
 559         optional => 1,
 560     },
 561     policy => {
 562         type => 'string',
 563         enum => [qw(preferred bind interleave)],
 564         description => "NUMA allocation policy.",
 565         optional => 1,
 566     },
 567 };
 568 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 569 my $numadesc = {
 570     optional => 1,
 571     type => 'string', format => $numa_fmt,
 572     description => "NUMA topology.",
 573 };
 574 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 575
 576 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 577     $confdesc->{"numa$i"} = $numadesc;
 578 }
 579
 580 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 581                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 582                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 583 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 584
 585 my $net_fmt_bridge_descr = <<__EOD__;
 586 Bridge to attach the network device to. The Proxmox VE standard bridge
 587 is called 'vmbr0'.
 588
 589 If you do not specify a bridge, we create a kvm user (NATed) network
 590 device, which provides DHCP and DNS services. The following addresses
 591 are used:
 592
 593  10.0.2.2   Gateway
 594  10.0.2.3   DNS Server
 595  10.0.2.4   SMB Server
 596
 597 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 598 __EOD__
 599
 600 my $net_fmt = {
 601     macaddr => {
 602         type => 'string',
 603         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 604         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 605         format_description => "XX:XX:XX:XX:XX:XX",
 606         optional => 1,
 607     },
 608     model => {
 609         type => 'string',
 610         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 611         enum => $nic_model_list,
 612         default_key => 1,
 613     },
 614     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 615     bridge => {
 616         type => 'string',
 617         description => $net_fmt_bridge_descr,
 618         format_description => 'bridge',
 619         optional => 1,
 620     },
 621     queues => {
 622         type => 'integer',
 623         minimum => 0, maximum => 16,
 624         description => 'Number of packet queues to be used on the device.',
 625         optional => 1,
 626     },
 627     rate => {
 628         type => 'number',
 629         minimum => 0,
 630         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 631         optional => 1,
 632     },
 633     tag => {
 634         type => 'integer',
 635         minimum => 1, maximum => 4094,
 636         description => 'VLAN tag to apply to packets on this interface.',
 637         optional => 1,
 638     },
 639     trunks => {
 640         type => 'string',
 641         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 642         description => 'VLAN trunks to pass through this interface.',
 643         format_description => 'vlanid[;vlanid...]',
 644         optional => 1,
 645     },
 646     firewall => {
 647         type => 'boolean',
 648         description => 'Whether this interface should be protected by the firewall.',
 649         optional => 1,
 650     },
 651     link_down => {
 652         type => 'boolean',
 653         description => 'Whether this interface should be disconnected (like pulling the plug).',
 654         optional => 1,
 655     },
 656 };
 657
 658 my $netdesc = {
 659     optional => 1,
 660     type => 'string', format => $net_fmt,
 661     description => "Specify network devices.",
 662 };
 663
 664 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 665
 666 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 667     $confdesc->{"net$i"} = $netdesc;
 668 }
 669
 670 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 671 sub verify_volume_id_or_qm_path {
 672     my ($volid, $noerr) = @_;
 673
 674     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 675         return $volid;
 676     }
 677
 678     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 679     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 680     if ($@) {
 681         return undef if $noerr;
 682         die $@;
 683     }
 684     return $volid;
 685 }
 686
 687 my $drivename_hash;
 688
 689 my %drivedesc_base = (
 690     volume => { alias => 'file' },
 691     file => {
 692         type => 'string',
 693         format => 'pve-volume-id-or-qm-path',
 694         default_key => 1,
 695         format_description => 'volume',
 696         description => "The drive's backing volume.",
 697     },
 698     media => {
 699         type => 'string',
 700         enum => [qw(cdrom disk)],
 701         description => "The drive's media type.",
 702         default => 'disk',
 703         optional => 1
 704     },
 705     cyls => {
 706         type => 'integer',
 707         description => "Force the drive's physical geometry to have a specific cylinder count.",
 708         optional => 1
 709     },
 710     heads => {
 711         type => 'integer',
 712         description => "Force the drive's physical geometry to have a specific head count.",
 713         optional => 1
 714     },
 715     secs => {
 716         type => 'integer',
 717         description => "Force the drive's physical geometry to have a specific sector count.",
 718         optional => 1
 719     },
 720     trans => {
 721         type => 'string',
 722         enum => [qw(none lba auto)],
 723         description => "Force disk geometry bios translation mode.",
 724         optional => 1,
 725     },
 726     snapshot => {
 727         type => 'boolean',
 728         description => "Whether the drive should be included when making snapshots.",
 729         optional => 1,
 730     },
 731     cache => {
 732         type => 'string',
 733         enum => [qw(none writethrough writeback unsafe directsync)],
 734         description => "The drive's cache mode",
 735         optional => 1,
 736     },
 737     format => get_standard_option('pve-qm-image-format'),
 738     size => {
 739         type => 'string',
 740         format => 'disk-size',
 741         format_description => 'DiskSize',
 742         description => "Disk size. This is purely informational and has no effect.",
 743         optional => 1,
 744     },
 745     backup => {
 746         type => 'boolean',
 747         description => "Whether the drive should be included when making backups.",
 748         optional => 1,
 749     },
 750     werror => {
 751         type => 'string',
 752         enum => [qw(enospc ignore report stop)],
 753         description => 'Write error action.',
 754         optional => 1,
 755     },
 756     aio => {
 757         type => 'string',
 758         enum => [qw(native threads)],
 759         description => 'AIO type to use.',
 760         optional => 1,
 761     },
 762     discard => {
 763         type => 'string',
 764         enum => [qw(ignore on)],
 765         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 766         optional => 1,
 767     },
 768     detect_zeroes => {
 769         type => 'boolean',
 770         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 771         optional => 1,
 772     },
 773     serial => {
 774         type => 'string',
 775         format => 'urlencoded',
 776         format_description => 'serial',
 777         maxLength => 20*3, # *3 since it's %xx url enoded
 778         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 779         optional => 1,
 780     }
 781 );
 782
 783 my %rerror_fmt = (
 784     rerror => {
 785         type => 'string',
 786         enum => [qw(ignore report stop)],
 787         description => 'Read error action.',
 788         optional => 1,
 789     },
 790 );
 791
 792 my %iothread_fmt = ( iothread => {
 793         type => 'boolean',
 794         description => "Whether to use iothreads for this drive",
 795         optional => 1,
 796 });
 797
 798 my %model_fmt = (
 799     model => {
 800         type => 'string',
 801         format => 'urlencoded',
 802         format_description => 'model',
 803         maxLength => 40*3, # *3 since it's %xx url enoded
 804         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 805         optional => 1,
 806     },
 807 );
 808
 809 my %queues_fmt = (
 810     queues => {
 811         type => 'integer',
 812         description => "Number of queues.",
 813         minimum => 2,
 814         optional => 1
 815     }
 816 );
 817
 818 my %scsiblock_fmt = (
 819     scsiblock => {
 820         type => 'boolean',
 821         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 822         optional => 1,
 823         default => 0,
 824     },
 825 );
 826
 827 my $add_throttle_desc = sub {
 828     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 829     my $d = {
 830         type => $type,
 831         format_description => $unit,
 832         description => "Maximum $what in $longunit.",
 833         optional => 1,
 834     };
 835     $d->{minimum} = $minimum if defined($minimum);
 836     $drivedesc_base{$key} = $d;
 837 };
 838 # throughput: (leaky bucket)
 839 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 840 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 841 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 842 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 843 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 844 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 845 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 846 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 847 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 848
 849 # pools: (pool of IO before throttling starts taking effect)
 850 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 851 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 852 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 853 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 854 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 855 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 856
 857 # burst lengths
 858 $add_throttle_desc->('bps_max_length',  'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 859 $add_throttle_desc->('bps_rd_length',   'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 860 $add_throttle_desc->('bps_wr_length',   'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 861 $add_throttle_desc->('iops_max_length', 'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 862 $add_throttle_desc->('iops_rd_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 863 $add_throttle_desc->('iops_wr_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 864
 865 my $ide_fmt = {
 866     %drivedesc_base,
 867     %rerror_fmt,
 868     %model_fmt,
 869 };
 870 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 871
 872 my $idedesc = {
 873     optional => 1,
 874     type => 'string', format => $ide_fmt,
 875     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 876 };
 877 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 878
 879 my $scsi_fmt = {
 880     %drivedesc_base,
 881     %iothread_fmt,
 882     %queues_fmt,
 883     %scsiblock_fmt,
 884 };
 885 my $scsidesc = {
 886     optional => 1,
 887     type => 'string', format => $scsi_fmt,
 888     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 889 };
 890 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 891
 892 my $sata_fmt = {
 893     %drivedesc_base,
 894     %rerror_fmt,
 895 };
 896 my $satadesc = {
 897     optional => 1,
 898     type => 'string', format => $sata_fmt,
 899     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 900 };
 901 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 902
 903 my $virtio_fmt = {
 904     %drivedesc_base,
 905     %iothread_fmt,
 906     %rerror_fmt,
 907 };
 908 my $virtiodesc = {
 909     optional => 1,
 910     type => 'string', format => $virtio_fmt,
 911     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 912 };
 913 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 914
 915 my $alldrive_fmt = {
 916     %drivedesc_base,
 917     %rerror_fmt,
 918     %iothread_fmt,
 919     %model_fmt,
 920     %queues_fmt,
 921     %scsiblock_fmt,
 922 };
 923
 924 my $efidisk_fmt = {
 925     volume => { alias => 'file' },
 926     file => {
 927         type => 'string',
 928         format => 'pve-volume-id-or-qm-path',
 929         default_key => 1,
 930         format_description => 'volume',
 931         description => "The drive's backing volume.",
 932     },
 933     format => get_standard_option('pve-qm-image-format'),
 934     size => {
 935         type => 'string',
 936         format => 'disk-size',
 937         format_description => 'DiskSize',
 938         description => "Disk size. This is purely informational and has no effect.",
 939         optional => 1,
 940     },
 941 };
 942
 943 my $efidisk_desc = {
 944     optional => 1,
 945     type => 'string', format => $efidisk_fmt,
 946     description => "Configure a Disk for storing EFI vars",
 947 };
 948
 949 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 950
 951 my $usb_fmt = {
 952     host => {
 953         default_key => 1,
 954         type => 'string', format => 'pve-qm-usb-device',
 955         format_description => 'HOSTUSBDEVICE|spice',
 956         description => <<EODESCR,
 957 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 958
 959  'bus-port(.port)*' (decimal numbers) or
 960  'vendor_id:product_id' (hexadeciaml numbers) or
 961  'spice'
 962
 963 You can use the 'lsusb -t' command to list existing usb devices.
 964
 965 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 966
 967 The value 'spice' can be used to add a usb redirection devices for spice.
 968 EODESCR
 969     },
 970     usb3 => {
 971         optional => 1,
 972         type => 'boolean',
 973         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 974         default => 0,
 975     },
 976 };
 977
 978 my $usbdesc = {
 979     optional => 1,
 980     type => 'string', format => $usb_fmt,
 981     description => "Configure an USB device (n is 0 to 4).",
 982 };
 983 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 984
 985 # NOTE: the match-groups of this regex are used in parse_hostpci
 986 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 987 my $hostpci_fmt = {
 988     host => {
 989         default_key => 1,
 990         type => 'string',
 991         pattern => qr/$PCIRE(;$PCIRE)*/,
 992         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 993         description => <<EODESCR,
 994 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 995 of PCI virtual functions of the host. HOSTPCIID syntax is:
 996
 997 'bus:dev.func' (hexadecimal numbers)
 998
 999 You can us the 'lspci' command to list existing PCI devices.
1000 EODESCR
1001     },
1002     rombar => {
1003         type => 'boolean',
1004         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1005         optional => 1,
1006         default => 1,
1007     },
1008     romfile => {
1009         type => 'string',
1010         pattern => '[^,;]+',
1011         format_description => 'string',
1012         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1013         optional => 1,
1014     },
1015     pcie => {
1016         type => 'boolean',
1017         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1018         optional => 1,
1019         default => 0,
1020     },
1021     'x-vga' => {
1022         type => 'boolean',
1023         description =>  "Enable vfio-vga device support.",
1024         optional => 1,
1025         default => 0,
1026     },
1027 };
1028 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1029
1030 my $hostpcidesc = {
1031         optional => 1,
1032         type => 'string', format => 'pve-qm-hostpci',
1033         description => "Map host PCI devices into guest.",
1034         verbose_description =>  <<EODESCR,
1035 Map host PCI devices into guest.
1036
1037 NOTE: This option allows direct access to host hardware. So it is no longer
1038 possible to migrate such machines - use with special care.
1039
1040 CAUTION: Experimental! User reported problems with this option.
1041 EODESCR
1042 };
1043 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1044
1045 my $serialdesc = {
1046         optional => 1,
1047         type => 'string',
1048         pattern => '(/dev/.+|socket)',
1049         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1050         verbose_description =>  <<EODESCR,
1051 Create a serial device inside the VM (n is 0 to 3), and pass through a
1052 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1053 host side (use 'qm terminal' to open a terminal connection).
1054
1055 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1056
1057 CAUTION: Experimental! User reported problems with this option.
1058 EODESCR
1059 };
1060
1061 my $paralleldesc= {
1062         optional => 1,
1063         type => 'string',
1064         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1065         description =>  "Map host parallel devices (n is 0 to 2).",
1066         verbose_description =>  <<EODESCR,
1067 Map host parallel devices (n is 0 to 2).
1068
1069 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1070
1071 CAUTION: Experimental! User reported problems with this option.
1072 EODESCR
1073 };
1074
1075 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1076     $confdesc->{"parallel$i"} = $paralleldesc;
1077 }
1078
1079 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1080     $confdesc->{"serial$i"} = $serialdesc;
1081 }
1082
1083 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1084     $confdesc->{"hostpci$i"} = $hostpcidesc;
1085 }
1086
1087 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1088     $drivename_hash->{"ide$i"} = 1;
1089     $confdesc->{"ide$i"} = $idedesc;
1090 }
1091
1092 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1093     $drivename_hash->{"sata$i"} = 1;
1094     $confdesc->{"sata$i"} = $satadesc;
1095 }
1096
1097 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1098     $drivename_hash->{"scsi$i"} = 1;
1099     $confdesc->{"scsi$i"} = $scsidesc ;
1100 }
1101
1102 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1103     $drivename_hash->{"virtio$i"} = 1;
1104     $confdesc->{"virtio$i"} = $virtiodesc;
1105 }
1106
1107 $drivename_hash->{efidisk0} = 1;
1108 $confdesc->{efidisk0} = $efidisk_desc;
1109
1110 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1111     $confdesc->{"usb$i"} = $usbdesc;
1112 }
1113
1114 my $unuseddesc = {
1115     optional => 1,
1116     type => 'string', format => 'pve-volume-id',
1117     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1118 };
1119
1120 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1121     $confdesc->{"unused$i"} = $unuseddesc;
1122 }
1123
1124 my $kvm_api_version = 0;
1125
1126 sub kvm_version {
1127
1128     return $kvm_api_version if $kvm_api_version;
1129
1130     my $fh = IO::File->new("</dev/kvm") ||
1131         return 0;
1132
1133     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1134         $kvm_api_version = $v;
1135     }
1136
1137     $fh->close();
1138
1139     return  $kvm_api_version;
1140 }
1141
1142 my $kvm_user_version;
1143
1144 sub kvm_user_version {
1145
1146     return $kvm_user_version if $kvm_user_version;
1147
1148     $kvm_user_version = 'unknown';
1149
1150     my $code = sub {
1151         my $line = shift;
1152         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1153             $kvm_user_version = $2;
1154         }
1155     };
1156
1157     eval { run_command("kvm -version", outfunc => $code); };
1158     warn $@ if $@;
1159
1160     return $kvm_user_version;
1161
1162 }
1163
1164 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1165
1166 sub valid_drive_names {
1167     # order is important - used to autoselect boot disk
1168     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1169             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1170             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1171             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1172             'efidisk0');
1173 }
1174
1175 sub is_valid_drivename {
1176     my $dev = shift;
1177
1178     return defined($drivename_hash->{$dev});
1179 }
1180
1181 sub option_exists {
1182     my $key = shift;
1183     return defined($confdesc->{$key});
1184 }
1185
1186 sub nic_models {
1187     return $nic_model_list;
1188 }
1189
1190 sub os_list_description {
1191
1192     return {
1193         other => 'Other',
1194         wxp => 'Windows XP',
1195         w2k => 'Windows 2000',
1196         w2k3 =>, 'Windows 2003',
1197         w2k8 => 'Windows 2008',
1198         wvista => 'Windows Vista',
1199         win7 => 'Windows 7',
1200         win8 => 'Windows 8/2012',
1201         win10 => 'Windows 10/2016',
1202         l24 => 'Linux 2.4',
1203         l26 => 'Linux 2.6',
1204     };
1205 }
1206
1207 my $cdrom_path;
1208
1209 sub get_cdrom_path {
1210
1211     return  $cdrom_path if $cdrom_path;
1212
1213     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1214     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1215     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1216 }
1217
1218 sub get_iso_path {
1219     my ($storecfg, $vmid, $cdrom) = @_;
1220
1221     if ($cdrom eq 'cdrom') {
1222         return get_cdrom_path();
1223     } elsif ($cdrom eq 'none') {
1224         return '';
1225     } elsif ($cdrom =~ m|^/|) {
1226         return $cdrom;
1227     } else {
1228         return PVE::Storage::path($storecfg, $cdrom);
1229     }
1230 }
1231
1232 # try to convert old style file names to volume IDs
1233 sub filename_to_volume_id {
1234     my ($vmid, $file, $media) = @_;
1235
1236     if (!($file eq 'none' || $file eq 'cdrom' ||
1237           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1238
1239         return undef if $file =~ m|/|;
1240
1241         if ($media && $media eq 'cdrom') {
1242             $file = "local:iso/$file";
1243         } else {
1244             $file = "local:$vmid/$file";
1245         }
1246     }
1247
1248     return $file;
1249 }
1250
1251 sub verify_media_type {
1252     my ($opt, $vtype, $media) = @_;
1253
1254     return if !$media;
1255
1256     my $etype;
1257     if ($media eq 'disk') {
1258         $etype = 'images';
1259     } elsif ($media eq 'cdrom') {
1260         $etype = 'iso';
1261     } else {
1262         die "internal error";
1263     }
1264
1265     return if ($vtype eq $etype);
1266
1267     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1268 }
1269
1270 sub cleanup_drive_path {
1271     my ($opt, $storecfg, $drive) = @_;
1272
1273     # try to convert filesystem paths to volume IDs
1274
1275     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1276         ($drive->{file} !~ m|^/dev/.+|) &&
1277         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1278         ($drive->{file} !~ m/^\d+$/)) {
1279         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1280         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1281         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1282         verify_media_type($opt, $vtype, $drive->{media});
1283         $drive->{file} = $volid;
1284     }
1285
1286     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1287 }
1288
1289 sub parse_hotplug_features {
1290     my ($data) = @_;
1291
1292     my $res = {};
1293
1294     return $res if $data eq '0';
1295
1296     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1297
1298     foreach my $feature (PVE::Tools::split_list($data)) {
1299         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1300             $res->{$1} = 1;
1301         } else {
1302             die "invalid hotplug feature '$feature'\n";
1303         }
1304     }
1305     return $res;
1306 }
1307
1308 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1309 sub pve_verify_hotplug_features {
1310     my ($value, $noerr) = @_;
1311
1312     return $value if parse_hotplug_features($value);
1313
1314     return undef if $noerr;
1315
1316     die "unable to parse hotplug option\n";
1317 }
1318
1319 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1320 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1321 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1322 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1323 #        [,iothread=on][,serial=serial][,model=model]
1324
1325 sub parse_drive {
1326     my ($key, $data) = @_;
1327
1328     my ($interface, $index);
1329
1330     if ($key =~ m/^([^\d]+)(\d+)$/) {
1331         $interface = $1;
1332         $index = $2;
1333     } else {
1334         return undef;
1335     }
1336
1337     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1338                                      : $confdesc->{$key}->{format};
1339     if (!$desc) {
1340         warn "invalid drive key: $key\n";
1341         return undef;
1342     }
1343     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1344     return undef if !$res;
1345     $res->{interface} = $interface;
1346     $res->{index} = $index;
1347
1348     my $error = 0;
1349     foreach my $opt (qw(bps bps_rd bps_wr)) {
1350         if (my $bps = defined(delete $res->{$opt})) {
1351             if (defined($res->{"m$opt"})) {
1352                 warn "both $opt and m$opt specified\n";
1353                 ++$error;
1354                 next;
1355             }
1356             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1357         }
1358     }
1359
1360     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1361     for my $requirement (
1362         [bps_max_length => 'mbps_max'],
1363         [bps_rd_max_length => 'mbps_rd_max'],
1364         [bps_wr_max_length => 'mbps_wr_max'],
1365         [iops_max_length => 'iops_max'],
1366         [iops_rd_max_length => 'iops_rd_max'],
1367         [iops_wr_max_length => 'iops_wr_max']) {
1368         my ($option, $requires) = @$requirement;
1369         if ($res->{$option} && !$res->{$requires}) {
1370             warn "$option requires $requires\n";
1371             ++$error;
1372         }
1373     }
1374
1375     return undef if $error;
1376
1377     return undef if $res->{mbps_rd} && $res->{mbps};
1378     return undef if $res->{mbps_wr} && $res->{mbps};
1379     return undef if $res->{iops_rd} && $res->{iops};
1380     return undef if $res->{iops_wr} && $res->{iops};
1381
1382     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1383         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1384         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1385         return undef if $res->{interface} eq 'virtio';
1386     }
1387
1388     if (my $size = $res->{size}) {
1389         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1390     }
1391
1392     return $res;
1393 }
1394
1395 sub print_drive {
1396     my ($vmid, $drive) = @_;
1397     my $data = { %$drive };
1398     delete $data->{$_} for qw(index interface);
1399     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1400 }
1401
1402 sub scsi_inquiry {
1403     my($fh, $noerr) = @_;
1404
1405     my $SG_IO = 0x2285;
1406     my $SG_GET_VERSION_NUM = 0x2282;
1407
1408     my $versionbuf = "\x00" x 8;
1409     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1410     if (!$ret) {
1411         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1412         return undef;
1413     }
1414     my $version = unpack("I", $versionbuf);
1415     if ($version < 30000) {
1416         die "scsi generic interface too old\n"  if !$noerr;
1417         return undef;
1418     }
1419
1420     my $buf = "\x00" x 36;
1421     my $sensebuf = "\x00" x 8;
1422     my $cmd = pack("C x3 C x1", 0x12, 36);
1423
1424     # see /usr/include/scsi/sg.h
1425     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1426
1427     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1428                       length($sensebuf), 0, length($buf), $buf,
1429                       $cmd, $sensebuf, 6000);
1430
1431     $ret = ioctl($fh, $SG_IO, $packet);
1432     if (!$ret) {
1433         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1434         return undef;
1435     }
1436
1437     my @res = unpack($sg_io_hdr_t, $packet);
1438     if ($res[17] || $res[18]) {
1439         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1440         return undef;
1441     }
1442
1443     my $res = {};
1444     (my $byte0, my $byte1, $res->{vendor},
1445      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1446
1447     $res->{removable} = $byte1 & 128 ? 1 : 0;
1448     $res->{type} = $byte0 & 31;
1449
1450     return $res;
1451 }
1452
1453 sub path_is_scsi {
1454     my ($path) = @_;
1455
1456     my $fh = IO::File->new("+<$path") || return undef;
1457     my $res = scsi_inquiry($fh, 1);
1458     close($fh);
1459
1460     return $res;
1461 }
1462
1463 sub machine_type_is_q35 {
1464     my ($conf) = @_;
1465
1466     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1467 }
1468
1469 sub print_tabletdevice_full {
1470     my ($conf) = @_;
1471
1472     my $q35 = machine_type_is_q35($conf);
1473
1474     # we use uhci for old VMs because tablet driver was buggy in older qemu
1475     my $usbbus = $q35 ? "ehci" : "uhci";
1476
1477     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1478 }
1479
1480 sub print_drivedevice_full {
1481     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1482
1483     my $device = '';
1484     my $maxdev = 0;
1485
1486     if ($drive->{interface} eq 'virtio') {
1487         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1488         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1489         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1490     } elsif ($drive->{interface} eq 'scsi') {
1491
1492         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1493         my $unit = $drive->{index} % $maxdev;
1494         my $devicetype = 'hd';
1495         my $path = '';
1496         if (drive_is_cdrom($drive)) {
1497             $devicetype = 'cd';
1498         } else {
1499             if ($drive->{file} =~ m|^/|) {
1500                 $path = $drive->{file};
1501                 if (my $info = path_is_scsi($path)) {
1502                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1503                         $devicetype = 'block';
1504                     } elsif ($info->{type} == 1) { # tape
1505                         $devicetype = 'generic';
1506                     }
1507                 }
1508             } else {
1509                  $path = PVE::Storage::path($storecfg, $drive->{file});
1510             }
1511
1512             if($path =~ m/^iscsi\:\/\//){
1513                 $devicetype = 'generic';
1514             }
1515         }
1516
1517         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1518             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1519         } else {
1520             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1521         }
1522
1523     } elsif ($drive->{interface} eq 'ide'){
1524         $maxdev = 2;
1525         my $controller = int($drive->{index} / $maxdev);
1526         my $unit = $drive->{index} % $maxdev;
1527         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1528
1529         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1530         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1531             $model = URI::Escape::uri_unescape($model);
1532             $device .= ",model=$model";
1533         }
1534     } elsif ($drive->{interface} eq 'sata'){
1535         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1536         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1537         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1538     } elsif ($drive->{interface} eq 'usb') {
1539         die "implement me";
1540         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1541     } else {
1542         die "unsupported interface type";
1543     }
1544
1545     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1546
1547     return $device;
1548 }
1549
1550 sub get_initiator_name {
1551     my $initiator;
1552
1553     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1554     while (defined(my $line = <$fh>)) {
1555         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1556         $initiator = $1;
1557         last;
1558     }
1559     $fh->close();
1560
1561     return $initiator;
1562 }
1563
1564 sub print_drive_full {
1565     my ($storecfg, $vmid, $drive) = @_;
1566
1567     my $path;
1568     my $volid = $drive->{file};
1569     my $format;
1570
1571     if (drive_is_cdrom($drive)) {
1572         $path = get_iso_path($storecfg, $vmid, $volid);
1573     } else {
1574         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1575         if ($storeid) {
1576             $path = PVE::Storage::path($storecfg, $volid);
1577             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1578             $format = qemu_img_format($scfg, $volname);
1579         } else {
1580             $path = $volid;
1581             $format = "raw";
1582         }
1583    }
1584
1585     my $opts = '';
1586     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1587     foreach my $o (@qemu_drive_options) {
1588         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1589     }
1590     if (my $serial = $drive->{serial}) {
1591         $serial = URI::Escape::uri_unescape($serial);
1592         $opts .= ",serial=$serial";
1593     }
1594
1595     $opts .= ",format=$format" if $format && !$drive->{format};
1596
1597     foreach my $o (qw(bps bps_rd bps_wr)) {
1598         my $v = $drive->{"m$o"};
1599         $opts .= ",$o=" . int($v*1024*1024) if $v;
1600     }
1601
1602     my $cache_direct = 0;
1603
1604     if (my $cache = $drive->{cache}) {
1605         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1606     } elsif (!drive_is_cdrom($drive)) {
1607         $opts .= ",cache=none";
1608         $cache_direct = 1;
1609     }
1610
1611     # aio native works only with O_DIRECT
1612     if (!$drive->{aio}) {
1613         if($cache_direct) {
1614             $opts .= ",aio=native";
1615         } else {
1616             $opts .= ",aio=threads";
1617         }
1618     }
1619
1620     if (!drive_is_cdrom($drive)) {
1621         my $detectzeroes;
1622         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1623             $detectzeroes = 'off';
1624         } elsif ($drive->{discard}) {
1625             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1626         } else {
1627             # This used to be our default with discard not being specified:
1628             $detectzeroes = 'on';
1629         }
1630         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1631     }
1632
1633     my $pathinfo = $path ? "file=$path," : '';
1634
1635     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1636 }
1637
1638 sub print_netdevice_full {
1639     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1640
1641     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1642
1643     my $device = $net->{model};
1644     if ($net->{model} eq 'virtio') {
1645          $device = 'virtio-net-pci';
1646      };
1647
1648     my $pciaddr = print_pci_addr("$netid", $bridges);
1649     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1650     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1651         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1652         my $vectors = $net->{queues} * 2 + 2;
1653         $tmpstr .= ",vectors=$vectors,mq=on";
1654     }
1655     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1656
1657     if ($use_old_bios_files) {
1658         my $romfile;
1659         if ($device eq 'virtio-net-pci') {
1660             $romfile = 'pxe-virtio.rom';
1661         } elsif ($device eq 'e1000') {
1662             $romfile = 'pxe-e1000.rom';
1663         } elsif ($device eq 'ne2k') {
1664             $romfile = 'pxe-ne2k_pci.rom';
1665         } elsif ($device eq 'pcnet') {
1666             $romfile = 'pxe-pcnet.rom';
1667         } elsif ($device eq 'rtl8139') {
1668             $romfile = 'pxe-rtl8139.rom';
1669         }
1670         $tmpstr .= ",romfile=$romfile" if $romfile;
1671     }
1672
1673     return $tmpstr;
1674 }
1675
1676 sub print_netdev_full {
1677     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1678
1679     my $i = '';
1680     if ($netid =~ m/^net(\d+)$/) {
1681         $i = int($1);
1682     }
1683
1684     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1685
1686     my $ifname = "tap${vmid}i$i";
1687
1688     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1689     die "interface name '$ifname' is too long (max 15 character)\n"
1690         if length($ifname) >= 16;
1691
1692     my $vhostparam = '';
1693     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1694
1695     my $vmname = $conf->{name} || "vm$vmid";
1696
1697     my $netdev = "";
1698     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1699
1700     if ($net->{bridge}) {
1701         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1702     } else {
1703         $netdev = "type=user,id=$netid,hostname=$vmname";
1704     }
1705
1706     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1707
1708     return $netdev;
1709 }
1710
1711
1712 sub print_cpu_device {
1713     my ($conf, $id) = @_;
1714
1715     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
1716     my $cpu = $nokvm ? "qemu64" : "kvm64";
1717     if (my $cputype = $conf->{cpu}) {
1718         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1719             or die "Cannot parse cpu description: $cputype\n";
1720         $cpu = $cpuconf->{cputype};
1721     }
1722
1723     my $sockets = 1;
1724     $sockets = $conf->{sockets} if  $conf->{sockets};
1725     my $cores = $conf->{cores} || 1;
1726
1727     my $current_core = ($id - 1) % $cores;
1728     my $current_socket = int(($id - $current_core)/$cores);
1729
1730     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1731 }
1732
1733 sub drive_is_cdrom {
1734     my ($drive) = @_;
1735
1736     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1737
1738 }
1739
1740 sub parse_number_sets {
1741     my ($set) = @_;
1742     my $res = [];
1743     foreach my $part (split(/;/, $set)) {
1744         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1745             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1746             push @$res, [ $1, $2 ];
1747         } else {
1748             die "invalid range: $part\n";
1749         }
1750     }
1751     return $res;
1752 }
1753
1754 sub parse_numa {
1755     my ($data) = @_;
1756
1757     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1758     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1759     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1760     return $res;
1761 }
1762
1763 sub parse_hostpci {
1764     my ($value) = @_;
1765
1766     return undef if !$value;
1767
1768     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1769
1770     my @idlist = split(/;/, $res->{host});
1771     delete $res->{host};
1772     foreach my $id (@idlist) {
1773         if ($id =~ /^$PCIRE$/) {
1774             if (defined($2)) {
1775                 push @{$res->{pciid}}, { id => $1, function => $2 };
1776             } else {
1777                 my $pcidevices = lspci($1);
1778                 $res->{pciid} = $pcidevices->{$1};
1779             }
1780         } else {
1781             # should have been caught by parse_property_string already
1782             die "failed to parse PCI id: $id\n";
1783         }
1784     }
1785     return $res;
1786 }
1787
1788 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1789 sub parse_net {
1790     my ($data) = @_;
1791
1792     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1793     if ($@) {
1794         warn $@;
1795         return undef;
1796     }
1797     if (!defined($res->{macaddr})) {
1798         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1799         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1800     }
1801     return $res;
1802 }
1803
1804 sub print_net {
1805     my $net = shift;
1806
1807     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1808 }
1809
1810 sub add_random_macs {
1811     my ($settings) = @_;
1812
1813     foreach my $opt (keys %$settings) {
1814         next if $opt !~ m/^net(\d+)$/;
1815         my $net = parse_net($settings->{$opt});
1816         next if !$net;
1817         $settings->{$opt} = print_net($net);
1818     }
1819 }
1820
1821 sub vm_is_volid_owner {
1822     my ($storecfg, $vmid, $volid) = @_;
1823
1824     if ($volid !~  m|^/|) {
1825         my ($path, $owner);
1826         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1827         if ($owner && ($owner == $vmid)) {
1828             return 1;
1829         }
1830     }
1831
1832     return undef;
1833 }
1834
1835 sub split_flagged_list {
1836     my $text = shift || '';
1837     $text =~ s/[,;]/ /g;
1838     $text =~ s/^\s+//;
1839     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1840 }
1841
1842 sub join_flagged_list {
1843     my ($how, $lst) = @_;
1844     join $how, map { $lst->{$_} . $_ } keys %$lst;
1845 }
1846
1847 sub vmconfig_delete_pending_option {
1848     my ($conf, $key, $force) = @_;
1849
1850     delete $conf->{pending}->{$key};
1851     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1852     $pending_delete_hash->{$key} = $force ? '!' : '';
1853     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1854 }
1855
1856 sub vmconfig_undelete_pending_option {
1857     my ($conf, $key) = @_;
1858
1859     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1860     delete $pending_delete_hash->{$key};
1861
1862     if (%$pending_delete_hash) {
1863         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1864     } else {
1865         delete $conf->{pending}->{delete};
1866     }
1867 }
1868
1869 sub vmconfig_register_unused_drive {
1870     my ($storecfg, $vmid, $conf, $drive) = @_;
1871
1872     if (!drive_is_cdrom($drive)) {
1873         my $volid = $drive->{file};
1874         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1875             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1876         }
1877     }
1878 }
1879
1880 sub vmconfig_cleanup_pending {
1881     my ($conf) = @_;
1882
1883     # remove pending changes when nothing changed
1884     my $changes;
1885     foreach my $opt (keys %{$conf->{pending}}) {
1886         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1887             $changes = 1;
1888             delete $conf->{pending}->{$opt};
1889         }
1890     }
1891
1892     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1893     my $pending_delete_hash = {};
1894     while (my ($opt, $force) = each %$current_delete_hash) {
1895         if (defined($conf->{$opt})) {
1896             $pending_delete_hash->{$opt} = $force;
1897         } else {
1898             $changes = 1;
1899         }
1900     }
1901
1902     if (%$pending_delete_hash) {
1903         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1904     } else {
1905         delete $conf->{pending}->{delete};
1906     }
1907
1908     return $changes;
1909 }
1910
1911 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1912 my $smbios1_fmt = {
1913     uuid => {
1914         type => 'string',
1915         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1916         format_description => 'UUID',
1917         description => "Set SMBIOS1 UUID.",
1918         optional => 1,
1919     },
1920     version => {
1921         type => 'string',
1922         pattern => '\S+',
1923         format_description => 'string',
1924         description => "Set SMBIOS1 version.",
1925         optional => 1,
1926     },
1927     serial => {
1928         type => 'string',
1929         pattern => '\S+',
1930         format_description => 'string',
1931         description => "Set SMBIOS1 serial number.",
1932         optional => 1,
1933     },
1934     manufacturer => {
1935         type => 'string',
1936         pattern => '\S+',
1937         format_description => 'string',
1938         description => "Set SMBIOS1 manufacturer.",
1939         optional => 1,
1940     },
1941     product => {
1942         type => 'string',
1943         pattern => '\S+',
1944         format_description => 'string',
1945         description => "Set SMBIOS1 product ID.",
1946         optional => 1,
1947     },
1948     sku => {
1949         type => 'string',
1950         pattern => '\S+',
1951         format_description => 'string',
1952         description => "Set SMBIOS1 SKU string.",
1953         optional => 1,
1954     },
1955     family => {
1956         type => 'string',
1957         pattern => '\S+',
1958         format_description => 'string',
1959         description => "Set SMBIOS1 family string.",
1960         optional => 1,
1961     },
1962 };
1963
1964 sub parse_smbios1 {
1965     my ($data) = @_;
1966
1967     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1968     warn $@ if $@;
1969     return $res;
1970 }
1971
1972 sub print_smbios1 {
1973     my ($smbios1) = @_;
1974     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1975 }
1976
1977 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1978
1979 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1980 sub verify_bootdisk {
1981     my ($value, $noerr) = @_;
1982
1983     return $value if is_valid_drivename($value);
1984
1985     return undef if $noerr;
1986
1987     die "invalid boot disk '$value'\n";
1988 }
1989
1990 sub parse_watchdog {
1991     my ($value) = @_;
1992
1993     return undef if !$value;
1994
1995     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1996     warn $@ if $@;
1997     return $res;
1998 }
1999
2000 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2001 sub verify_usb_device {
2002     my ($value, $noerr) = @_;
2003
2004     return $value if parse_usb_device($value);
2005
2006     return undef if $noerr;
2007
2008     die "unable to parse usb device\n";
2009 }
2010
2011 # add JSON properties for create and set function
2012 sub json_config_properties {
2013     my $prop = shift;
2014
2015     foreach my $opt (keys %$confdesc) {
2016         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2017         $prop->{$opt} = $confdesc->{$opt};
2018     }
2019
2020     return $prop;
2021 }
2022
2023 sub check_type {
2024     my ($key, $value) = @_;
2025
2026     die "unknown setting '$key'\n" if !$confdesc->{$key};
2027
2028     my $type = $confdesc->{$key}->{type};
2029
2030     if (!defined($value)) {
2031         die "got undefined value\n";
2032     }
2033
2034     if ($value =~ m/[\n\r]/) {
2035         die "property contains a line feed\n";
2036     }
2037
2038     if ($type eq 'boolean') {
2039         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2040         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2041         die "type check ('boolean') failed - got '$value'\n";
2042     } elsif ($type eq 'integer') {
2043         return int($1) if $value =~ m/^(\d+)$/;
2044         die "type check ('integer') failed - got '$value'\n";
2045     } elsif ($type eq 'number') {
2046         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2047         die "type check ('number') failed - got '$value'\n";
2048     } elsif ($type eq 'string') {
2049         if (my $fmt = $confdesc->{$key}->{format}) {
2050             PVE::JSONSchema::check_format($fmt, $value);
2051             return $value;
2052         }
2053         $value =~ s/^\"(.*)\"$/$1/;
2054         return $value;
2055     } else {
2056         die "internal error"
2057     }
2058 }
2059
2060 sub check_iommu_support{
2061     #fixme : need to check IOMMU support
2062     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2063
2064     my $iommu=1;
2065     return $iommu;
2066
2067 }
2068
2069 sub touch_config {
2070     my ($vmid) = @_;
2071
2072     my $conf = PVE::QemuConfig->config_file($vmid);
2073     utime undef, undef, $conf;
2074 }
2075
2076 sub destroy_vm {
2077     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2078
2079     my $conffile = PVE::QemuConfig->config_file($vmid);
2080
2081     my $conf = PVE::QemuConfig->load_config($vmid);
2082
2083     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2084
2085     # only remove disks owned by this VM
2086     foreach_drive($conf, sub {
2087         my ($ds, $drive) = @_;
2088
2089         return if drive_is_cdrom($drive);
2090
2091         my $volid = $drive->{file};
2092
2093         return if !$volid || $volid =~ m|^/|;
2094
2095         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2096         return if !$path || !$owner || ($owner != $vmid);
2097
2098         eval {
2099             PVE::Storage::vdisk_free($storecfg, $volid);
2100         };
2101         warn "Could not remove disk '$volid', check manually: $@" if $@;
2102
2103     });
2104
2105     if ($keep_empty_config) {
2106         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2107     } else {
2108         unlink $conffile;
2109     }
2110
2111     # also remove unused disk
2112     eval {
2113         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2114
2115         eval {
2116             PVE::Storage::foreach_volid($dl, sub {
2117                 my ($volid, $sid, $volname, $d) = @_;
2118                 PVE::Storage::vdisk_free($storecfg, $volid);
2119             });
2120         };
2121         warn $@ if $@;
2122
2123     };
2124     warn $@ if $@;
2125 }
2126
2127 sub parse_vm_config {
2128     my ($filename, $raw) = @_;
2129
2130     return undef if !defined($raw);
2131
2132     my $res = {
2133         digest => Digest::SHA::sha1_hex($raw),
2134         snapshots => {},
2135         pending => {},
2136     };
2137
2138     $filename =~ m|/qemu-server/(\d+)\.conf$|
2139         || die "got strange filename '$filename'";
2140
2141     my $vmid = $1;
2142
2143     my $conf = $res;
2144     my $descr;
2145     my $section = '';
2146
2147     my @lines = split(/\n/, $raw);
2148     foreach my $line (@lines) {
2149         next if $line =~ m/^\s*$/;
2150
2151         if ($line =~ m/^\[PENDING\]\s*$/i) {
2152             $section = 'pending';
2153             if (defined($descr)) {
2154                 $descr =~ s/\s+$//;
2155                 $conf->{description} = $descr;
2156             }
2157             $descr = undef;
2158             $conf = $res->{$section} = {};
2159             next;
2160
2161         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2162             $section = $1;
2163             if (defined($descr)) {
2164                 $descr =~ s/\s+$//;
2165                 $conf->{description} = $descr;
2166             }
2167             $descr = undef;
2168             $conf = $res->{snapshots}->{$section} = {};
2169             next;
2170         }
2171
2172         if ($line =~ m/^\#(.*)\s*$/) {
2173             $descr = '' if !defined($descr);
2174             $descr .= PVE::Tools::decode_text($1) . "\n";
2175             next;
2176         }
2177
2178         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2179             $descr = '' if !defined($descr);
2180             $descr .= PVE::Tools::decode_text($2);
2181         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2182             $conf->{snapstate} = $1;
2183         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2184             my $key = $1;
2185             my $value = $2;
2186             $conf->{$key} = $value;
2187         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2188             my $value = $1;
2189             if ($section eq 'pending') {
2190                 $conf->{delete} = $value; # we parse this later
2191             } else {
2192                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2193             }
2194         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2195             my $key = $1;
2196             my $value = $2;
2197             eval { $value = check_type($key, $value); };
2198             if ($@) {
2199                 warn "vm $vmid - unable to parse value of '$key' - $@";
2200             } else {
2201                 $key = 'ide2' if $key eq 'cdrom';
2202                 my $fmt = $confdesc->{$key}->{format};
2203                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2204                     my $v = parse_drive($key, $value);
2205                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2206                         $v->{file} = $volid;
2207                         $value = print_drive($vmid, $v);
2208                     } else {
2209                         warn "vm $vmid - unable to parse value of '$key'\n";
2210                         next;
2211                     }
2212                 }
2213
2214                 $conf->{$key} = $value;
2215             }
2216         }
2217     }
2218
2219     if (defined($descr)) {
2220         $descr =~ s/\s+$//;
2221         $conf->{description} = $descr;
2222     }
2223     delete $res->{snapstate}; # just to be sure
2224
2225     return $res;
2226 }
2227
2228 sub write_vm_config {
2229     my ($filename, $conf) = @_;
2230
2231     delete $conf->{snapstate}; # just to be sure
2232
2233     if ($conf->{cdrom}) {
2234         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2235         $conf->{ide2} = $conf->{cdrom};
2236         delete $conf->{cdrom};
2237     }
2238
2239     # we do not use 'smp' any longer
2240     if ($conf->{sockets}) {
2241         delete $conf->{smp};
2242     } elsif ($conf->{smp}) {
2243         $conf->{sockets} = $conf->{smp};
2244         delete $conf->{cores};
2245         delete $conf->{smp};
2246     }
2247
2248     my $used_volids = {};
2249
2250     my $cleanup_config = sub {
2251         my ($cref, $pending, $snapname) = @_;
2252
2253         foreach my $key (keys %$cref) {
2254             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2255                 $key eq 'snapstate' || $key eq 'pending';
2256             my $value = $cref->{$key};
2257             if ($key eq 'delete') {
2258                 die "propertry 'delete' is only allowed in [PENDING]\n"
2259                     if !$pending;
2260                 # fixme: check syntax?
2261                 next;
2262             }
2263             eval { $value = check_type($key, $value); };
2264             die "unable to parse value of '$key' - $@" if $@;
2265
2266             $cref->{$key} = $value;
2267
2268             if (!$snapname && is_valid_drivename($key)) {
2269                 my $drive = parse_drive($key, $value);
2270                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2271             }
2272         }
2273     };
2274
2275     &$cleanup_config($conf);
2276
2277     &$cleanup_config($conf->{pending}, 1);
2278
2279     foreach my $snapname (keys %{$conf->{snapshots}}) {
2280         die "internal error" if $snapname eq 'pending';
2281         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2282     }
2283
2284     # remove 'unusedX' settings if we re-add a volume
2285     foreach my $key (keys %$conf) {
2286         my $value = $conf->{$key};
2287         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2288             delete $conf->{$key};
2289         }
2290     }
2291
2292     my $generate_raw_config = sub {
2293         my ($conf, $pending) = @_;
2294
2295         my $raw = '';
2296
2297         # add description as comment to top of file
2298         if (defined(my $descr = $conf->{description})) {
2299             if ($descr) {
2300                 foreach my $cl (split(/\n/, $descr)) {
2301                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2302                 }
2303             } else {
2304                 $raw .= "#\n" if $pending;
2305             }
2306         }
2307
2308         foreach my $key (sort keys %$conf) {
2309             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2310             $raw .= "$key: $conf->{$key}\n";
2311         }
2312         return $raw;
2313     };
2314
2315     my $raw = &$generate_raw_config($conf);
2316
2317     if (scalar(keys %{$conf->{pending}})){
2318         $raw .= "\n[PENDING]\n";
2319         $raw .= &$generate_raw_config($conf->{pending}, 1);
2320     }
2321
2322     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2323         $raw .= "\n[$snapname]\n";
2324         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2325     }
2326
2327     return $raw;
2328 }
2329
2330 sub load_defaults {
2331
2332     my $res = {};
2333
2334     # we use static defaults from our JSON schema configuration
2335     foreach my $key (keys %$confdesc) {
2336         if (defined(my $default = $confdesc->{$key}->{default})) {
2337             $res->{$key} = $default;
2338         }
2339     }
2340
2341     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2342     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2343
2344     return $res;
2345 }
2346
2347 sub config_list {
2348     my $vmlist = PVE::Cluster::get_vmlist();
2349     my $res = {};
2350     return $res if !$vmlist || !$vmlist->{ids};
2351     my $ids = $vmlist->{ids};
2352
2353     foreach my $vmid (keys %$ids) {
2354         my $d = $ids->{$vmid};
2355         next if !$d->{node} || $d->{node} ne $nodename;
2356         next if !$d->{type} || $d->{type} ne 'qemu';
2357         $res->{$vmid}->{exists} = 1;
2358     }
2359     return $res;
2360 }
2361
2362 # test if VM uses local resources (to prevent migration)
2363 sub check_local_resources {
2364     my ($conf, $noerr) = @_;
2365
2366     my $loc_res = 0;
2367
2368     $loc_res = 1 if $conf->{hostusb}; # old syntax
2369     $loc_res = 1 if $conf->{hostpci}; # old syntax
2370
2371     foreach my $k (keys %$conf) {
2372         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2373         # sockets are safe: they will recreated be on the target side post-migrate
2374         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2375         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2376     }
2377
2378     die "VM uses local resources\n" if $loc_res && !$noerr;
2379
2380     return $loc_res;
2381 }
2382
2383 # check if used storages are available on all nodes (use by migrate)
2384 sub check_storage_availability {
2385     my ($storecfg, $conf, $node) = @_;
2386
2387     foreach_drive($conf, sub {
2388         my ($ds, $drive) = @_;
2389
2390         my $volid = $drive->{file};
2391         return if !$volid;
2392
2393         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2394         return if !$sid;
2395
2396         # check if storage is available on both nodes
2397         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2398         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2399    });
2400 }
2401
2402 # list nodes where all VM images are available (used by has_feature API)
2403 sub shared_nodes {
2404     my ($conf, $storecfg) = @_;
2405
2406     my $nodelist = PVE::Cluster::get_nodelist();
2407     my $nodehash = { map { $_ => 1 } @$nodelist };
2408     my $nodename = PVE::INotify::nodename();
2409
2410     foreach_drive($conf, sub {
2411         my ($ds, $drive) = @_;
2412
2413         my $volid = $drive->{file};
2414         return if !$volid;
2415
2416         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2417         if ($storeid) {
2418             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2419             if ($scfg->{disable}) {
2420                 $nodehash = {};
2421             } elsif (my $avail = $scfg->{nodes}) {
2422                 foreach my $node (keys %$nodehash) {
2423                     delete $nodehash->{$node} if !$avail->{$node};
2424                 }
2425             } elsif (!$scfg->{shared}) {
2426                 foreach my $node (keys %$nodehash) {
2427                     delete $nodehash->{$node} if $node ne $nodename
2428                 }
2429             }
2430         }
2431     });
2432
2433     return $nodehash
2434 }
2435
2436 sub check_cmdline {
2437     my ($pidfile, $pid) = @_;
2438
2439     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2440     if (defined($fh)) {
2441         my $line = <$fh>;
2442         $fh->close;
2443         return undef if !$line;
2444         my @param = split(/\0/, $line);
2445
2446         my $cmd = $param[0];
2447         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2448
2449         for (my $i = 0; $i < scalar (@param); $i++) {
2450             my $p = $param[$i];
2451             next if !$p;
2452             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2453                 my $p = $param[$i+1];
2454                 return 1 if $p && ($p eq $pidfile);
2455                 return undef;
2456             }
2457         }
2458     }
2459     return undef;
2460 }
2461
2462 sub check_running {
2463     my ($vmid, $nocheck, $node) = @_;
2464
2465     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2466
2467     die "unable to find configuration file for VM $vmid - no such machine\n"
2468         if !$nocheck && ! -f $filename;
2469
2470     my $pidfile = pidfile_name($vmid);
2471
2472     if (my $fd = IO::File->new("<$pidfile")) {
2473         my $st = stat($fd);
2474         my $line = <$fd>;
2475         close($fd);
2476
2477         my $mtime = $st->mtime;
2478         if ($mtime > time()) {
2479             warn "file '$filename' modified in future\n";
2480         }
2481
2482         if ($line =~ m/^(\d+)$/) {
2483             my $pid = $1;
2484             if (check_cmdline($pidfile, $pid)) {
2485                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2486                     return $pid;
2487                 }
2488             }
2489         }
2490     }
2491
2492     return undef;
2493 }
2494
2495 sub vzlist {
2496
2497     my $vzlist = config_list();
2498
2499     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2500
2501     while (defined(my $de = $fd->read)) {
2502         next if $de !~ m/^(\d+)\.pid$/;
2503         my $vmid = $1;
2504         next if !defined($vzlist->{$vmid});
2505         if (my $pid = check_running($vmid)) {
2506             $vzlist->{$vmid}->{pid} = $pid;
2507         }
2508     }
2509
2510     return $vzlist;
2511 }
2512
2513 sub disksize {
2514     my ($storecfg, $conf) = @_;
2515
2516     my $bootdisk = $conf->{bootdisk};
2517     return undef if !$bootdisk;
2518     return undef if !is_valid_drivename($bootdisk);
2519
2520     return undef if !$conf->{$bootdisk};
2521
2522     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2523     return undef if !defined($drive);
2524
2525     return undef if drive_is_cdrom($drive);
2526
2527     my $volid = $drive->{file};
2528     return undef if !$volid;
2529
2530     return $drive->{size};
2531 }
2532
2533 my $last_proc_pid_stat;
2534
2535 # get VM status information
2536 # This must be fast and should not block ($full == false)
2537 # We only query KVM using QMP if $full == true (this can be slow)
2538 sub vmstatus {
2539     my ($opt_vmid, $full) = @_;
2540
2541     my $res = {};
2542
2543     my $storecfg = PVE::Storage::config();
2544
2545     my $list = vzlist();
2546     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2547
2548     my $cpucount = $cpuinfo->{cpus} || 1;
2549
2550     foreach my $vmid (keys %$list) {
2551         next if $opt_vmid && ($vmid ne $opt_vmid);
2552
2553         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2554         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2555
2556         my $d = {};
2557         $d->{pid} = $list->{$vmid}->{pid};
2558
2559         # fixme: better status?
2560         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2561
2562         my $size = disksize($storecfg, $conf);
2563         if (defined($size)) {
2564             $d->{disk} = 0; # no info available
2565             $d->{maxdisk} = $size;
2566         } else {
2567             $d->{disk} = 0;
2568             $d->{maxdisk} = 0;
2569         }
2570
2571         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2572         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2573         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2574
2575         $d->{name} = $conf->{name} || "VM $vmid";
2576         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2577
2578         if ($conf->{balloon}) {
2579             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2580             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2581         }
2582
2583         $d->{uptime} = 0;
2584         $d->{cpu} = 0;
2585         $d->{mem} = 0;
2586
2587         $d->{netout} = 0;
2588         $d->{netin} = 0;
2589
2590         $d->{diskread} = 0;
2591         $d->{diskwrite} = 0;
2592
2593         $d->{template} = PVE::QemuConfig->is_template($conf);
2594
2595         $res->{$vmid} = $d;
2596     }
2597
2598     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2599     foreach my $dev (keys %$netdev) {
2600         next if $dev !~ m/^tap([1-9]\d*)i/;
2601         my $vmid = $1;
2602         my $d = $res->{$vmid};
2603         next if !$d;
2604
2605         $d->{netout} += $netdev->{$dev}->{receive};
2606         $d->{netin} += $netdev->{$dev}->{transmit};
2607
2608         if ($full) {
2609             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2610             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2611         }
2612
2613     }
2614
2615     my $ctime = gettimeofday;
2616
2617     foreach my $vmid (keys %$list) {
2618
2619         my $d = $res->{$vmid};
2620         my $pid = $d->{pid};
2621         next if !$pid;
2622
2623         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2624         next if !$pstat; # not running
2625
2626         my $used = $pstat->{utime} + $pstat->{stime};
2627
2628         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2629
2630         if ($pstat->{vsize}) {
2631             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2632         }
2633
2634         my $old = $last_proc_pid_stat->{$pid};
2635         if (!$old) {
2636             $last_proc_pid_stat->{$pid} = {
2637                 time => $ctime,
2638                 used => $used,
2639                 cpu => 0,
2640             };
2641             next;
2642         }
2643
2644         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2645
2646         if ($dtime > 1000) {
2647             my $dutime = $used -  $old->{used};
2648
2649             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2650             $last_proc_pid_stat->{$pid} = {
2651                 time => $ctime,
2652                 used => $used,
2653                 cpu => $d->{cpu},
2654             };
2655         } else {
2656             $d->{cpu} = $old->{cpu};
2657         }
2658     }
2659
2660     return $res if !$full;
2661
2662     my $qmpclient = PVE::QMPClient->new();
2663
2664     my $ballooncb = sub {
2665         my ($vmid, $resp) = @_;
2666
2667         my $info = $resp->{'return'};
2668         return if !$info->{max_mem};
2669
2670         my $d = $res->{$vmid};
2671
2672         # use memory assigned to VM
2673         $d->{maxmem} = $info->{max_mem};
2674         $d->{balloon} = $info->{actual};
2675
2676         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2677             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2678             $d->{freemem} = $info->{free_mem};
2679         }
2680
2681         $d->{ballooninfo} = $info;
2682     };
2683
2684     my $blockstatscb = sub {
2685         my ($vmid, $resp) = @_;
2686         my $data = $resp->{'return'} || [];
2687         my $totalrdbytes = 0;
2688         my $totalwrbytes = 0;
2689
2690         for my $blockstat (@$data) {
2691             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2692             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2693
2694             $blockstat->{device} =~ s/drive-//;
2695             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2696         }
2697         $res->{$vmid}->{diskread} = $totalrdbytes;
2698         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2699     };
2700
2701     my $statuscb = sub {
2702         my ($vmid, $resp) = @_;
2703
2704         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2705         # this fails if ballon driver is not loaded, so this must be
2706         # the last commnand (following command are aborted if this fails).
2707         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2708
2709         my $status = 'unknown';
2710         if (!defined($status = $resp->{'return'}->{status})) {
2711             warn "unable to get VM status\n";
2712             return;
2713         }
2714
2715         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2716     };
2717
2718     foreach my $vmid (keys %$list) {
2719         next if $opt_vmid && ($vmid ne $opt_vmid);
2720         next if !$res->{$vmid}->{pid}; # not running
2721         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2722     }
2723
2724     $qmpclient->queue_execute(undef, 2);
2725
2726     foreach my $vmid (keys %$list) {
2727         next if $opt_vmid && ($vmid ne $opt_vmid);
2728         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2729     }
2730
2731     return $res;
2732 }
2733
2734 sub foreach_drive {
2735     my ($conf, $func, @param) = @_;
2736
2737     foreach my $ds (valid_drive_names()) {
2738         next if !defined($conf->{$ds});
2739
2740         my $drive = parse_drive($ds, $conf->{$ds});
2741         next if !$drive;
2742
2743         &$func($ds, $drive, @param);
2744     }
2745 }
2746
2747 sub foreach_volid {
2748     my ($conf, $func, @param) = @_;
2749
2750     my $volhash = {};
2751
2752     my $test_volid = sub {
2753         my ($volid, $is_cdrom) = @_;
2754
2755         return if !$volid;
2756
2757         $volhash->{$volid} = $is_cdrom || 0;
2758     };
2759
2760     foreach_drive($conf, sub {
2761         my ($ds, $drive) = @_;
2762         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2763     });
2764
2765     foreach my $snapname (keys %{$conf->{snapshots}}) {
2766         my $snap = $conf->{snapshots}->{$snapname};
2767         &$test_volid($snap->{vmstate}, 0);
2768         foreach_drive($snap, sub {
2769             my ($ds, $drive) = @_;
2770             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2771         });
2772     }
2773
2774     foreach my $volid (keys %$volhash) {
2775         &$func($volid, $volhash->{$volid}, @param);
2776     }
2777 }
2778
2779 sub vga_conf_has_spice {
2780     my ($vga) = @_;
2781
2782     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2783
2784     return $1 || 1;
2785 }
2786
2787 sub config_to_command {
2788     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2789
2790     my $cmd = [];
2791     my $globalFlags = [];
2792     my $machineFlags = [];
2793     my $rtcFlags = [];
2794     my $cpuFlags = [];
2795     my $devices = [];
2796     my $pciaddr = '';
2797     my $bridges = {};
2798     my $kvmver = kvm_user_version();
2799     my $vernum = 0; # unknown
2800     my $ostype = $conf->{ostype};
2801     my $winversion = windows_version($ostype);
2802
2803     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2804         $vernum = $1*1000000+$2*1000;
2805     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2806         $vernum = $1*1000000+$2*1000+$3;
2807     }
2808
2809     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2810
2811     my $have_ovz = -f '/proc/vz/vestat';
2812
2813     my $q35 = machine_type_is_q35($conf);
2814     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2815     my $machine_type = $forcemachine || $conf->{machine};
2816     my $use_old_bios_files = undef;
2817     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2818
2819     my $cpuunits = defined($conf->{cpuunits}) ?
2820             $conf->{cpuunits} : $defaults->{cpuunits};
2821
2822     push @$cmd, '/usr/bin/kvm';
2823
2824     push @$cmd, '-id', $vmid;
2825
2826     my $use_virtio = 0;
2827
2828     my $qmpsocket = qmp_socket($vmid);
2829     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2830     push @$cmd, '-mon', "chardev=qmp,mode=control";
2831
2832
2833     push @$cmd, '-pidfile' , pidfile_name($vmid);
2834
2835     push @$cmd, '-daemonize';
2836
2837     if ($conf->{smbios1}) {
2838         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2839     }
2840
2841     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2842         my $ovmfbase;
2843
2844         # prefer the OVMF_CODE variant
2845         if (-f $OVMF_CODE) {
2846             $ovmfbase = $OVMF_CODE;
2847         } elsif (-f $OVMF_IMG) {
2848             $ovmfbase = $OVMF_IMG;
2849         }
2850
2851         die "no uefi base img found\n" if !$ovmfbase;
2852         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmfbase";
2853
2854         if (defined($conf->{efidisk0}) && ($ovmfbase eq $OVMF_CODE)) {
2855             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $conf->{efidisk0});
2856             my $format = $d->{format} // 'raw';
2857             my $path;
2858             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2859             if ($storeid) {
2860                 $path = PVE::Storage::path($storecfg, $d->{file});
2861                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2862                 $format = qemu_img_format($scfg, $volname);
2863             } else {
2864                 $path = $d->{file};
2865                 $format = "raw";
2866             }
2867             push @$cmd, '-drive', "if=pflash,unit=1,id=drive-efidisk0,format=$format,file=$path";
2868         } elsif ($ovmfbase eq $OVMF_CODE) {
2869             warn "using uefi without permanent efivars disk\n";
2870             my $ovmfvar_dst = "/tmp/$vmid-ovmf.fd";
2871             PVE::Tools::file_copy($OVMF_VARS, $ovmfvar_dst, 256*1024);
2872             push @$cmd, '-drive', "if=pflash,unit=1,format=raw,file=$ovmfvar_dst";
2873         } else {
2874             # if the base img is not OVMF_CODE, we do not have to bother
2875             # to create/use a vars image, since it will not be used anyway
2876             # this can only happen if someone manually deletes the OVMF_CODE image
2877             # or has an old pve-qemu-kvm version installed.
2878             # both should not happen, but we ignore it here
2879         }
2880     }
2881
2882
2883     # add usb controllers
2884     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2885     push @$devices, @usbcontrollers if @usbcontrollers;
2886     my $vga = $conf->{vga};
2887
2888     my $qxlnum = vga_conf_has_spice($vga);
2889     $vga = 'qxl' if $qxlnum;
2890
2891     if (!$vga) {
2892         $vga = $winversion >= 6 ? 'std' : 'cirrus';
2893     }
2894
2895     # enable absolute mouse coordinates (needed by vnc)
2896     my $tablet;
2897     if (defined($conf->{tablet})) {
2898         $tablet = $conf->{tablet};
2899     } else {
2900         $tablet = $defaults->{tablet};
2901         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2902         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2903     }
2904
2905     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2906
2907     my $kvm_off = 0;
2908     my $gpu_passthrough;
2909
2910     # host pci devices
2911     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2912         my $d = parse_hostpci($conf->{"hostpci$i"});
2913         next if !$d;
2914
2915         my $pcie = $d->{pcie};
2916         if($pcie){
2917             die "q35 machine model is not enabled" if !$q35;
2918             $pciaddr = print_pcie_addr("hostpci$i");
2919         }else{
2920             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2921         }
2922
2923         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2924         my $romfile = $d->{romfile};
2925
2926         my $xvga = '';
2927         if ($d->{'x-vga'}) {
2928             $xvga = ',x-vga=on';
2929             $kvm_off = 1;
2930             $vga = 'none';
2931             $gpu_passthrough = 1;
2932
2933             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2934                 $xvga = "";
2935             }
2936         }
2937         my $pcidevices = $d->{pciid};
2938         my $multifunction = 1 if @$pcidevices > 1;
2939
2940         my $j=0;
2941         foreach my $pcidevice (@$pcidevices) {
2942
2943             my $id = "hostpci$i";
2944             $id .= ".$j" if $multifunction;
2945             my $addr = $pciaddr;
2946             $addr .= ".$j" if $multifunction;
2947             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2948
2949             if($j == 0){
2950                 $devicestr .= "$rombar$xvga";
2951                 $devicestr .= ",multifunction=on" if $multifunction;
2952                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
2953             }
2954
2955             push @$devices, '-device', $devicestr;
2956             $j++;
2957         }
2958     }
2959
2960     # usb devices
2961     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
2962     push @$devices, @usbdevices if @usbdevices;
2963     # serial devices
2964     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2965         if (my $path = $conf->{"serial$i"}) {
2966             if ($path eq 'socket') {
2967                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2968                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2969                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2970             } else {
2971                 die "no such serial device\n" if ! -c $path;
2972                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2973                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2974             }
2975         }
2976     }
2977
2978     # parallel devices
2979     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2980         if (my $path = $conf->{"parallel$i"}) {
2981             die "no such parallel device\n" if ! -c $path;
2982             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2983             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2984             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2985         }
2986     }
2987
2988     my $vmname = $conf->{name} || "vm$vmid";
2989
2990     push @$cmd, '-name', $vmname;
2991
2992     my $sockets = 1;
2993     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2994     $sockets = $conf->{sockets} if  $conf->{sockets};
2995
2996     my $cores = $conf->{cores} || 1;
2997
2998     my $maxcpus = $sockets * $cores;
2999
3000     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3001
3002     my $allowed_vcpus = $cpuinfo->{cpus};
3003
3004     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3005         if ($allowed_vcpus < $maxcpus);
3006
3007     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3008
3009         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3010         for (my $i = 2; $i <= $vcpus; $i++)  {
3011             my $cpustr = print_cpu_device($conf,$i);
3012             push @$cmd, '-device', $cpustr;
3013         }
3014
3015     } else {
3016
3017         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3018     }
3019     push @$cmd, '-nodefaults';
3020
3021     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3022
3023     my $bootindex_hash = {};
3024     my $i = 1;
3025     foreach my $o (split(//, $bootorder)) {
3026         $bootindex_hash->{$o} = $i*100;
3027         $i++;
3028     }
3029
3030     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3031
3032     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3033
3034     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3035
3036     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3037
3038     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3039         my $socket = vnc_socket($vmid);
3040         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3041     } else {
3042         push @$cmd, '-nographic';
3043     }
3044
3045     # time drift fix
3046     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3047
3048     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3049     my $useLocaltime = $conf->{localtime};
3050
3051     if ($winversion >= 5) { # windows
3052         $useLocaltime = 1 if !defined($conf->{localtime});
3053
3054         # use time drift fix when acpi is enabled
3055         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3056             $tdf = 1 if !defined($conf->{tdf});
3057         }
3058     }
3059
3060     if ($winversion >= 6) {
3061         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3062         push @$cmd, '-no-hpet';
3063     }
3064
3065     push @$rtcFlags, 'driftfix=slew' if $tdf;
3066
3067     if ($nokvm) {
3068         push @$machineFlags, 'accel=tcg';
3069     } else {
3070         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3071     }
3072
3073     if ($machine_type) {
3074         push @$machineFlags, "type=${machine_type}";
3075     }
3076
3077     if ($conf->{startdate}) {
3078         push @$rtcFlags, "base=$conf->{startdate}";
3079     } elsif ($useLocaltime) {
3080         push @$rtcFlags, 'base=localtime';
3081     }
3082
3083     my $cpu = $nokvm ? "qemu64" : "kvm64";
3084     if (my $cputype = $conf->{cpu}) {
3085         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3086             or die "Cannot parse cpu description: $cputype\n";
3087         $cpu = $cpuconf->{cputype};
3088         $kvm_off = 1 if $cpuconf->{hidden};
3089     }
3090
3091     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3092
3093     push @$cpuFlags , '-x2apic'
3094         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3095
3096     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3097
3098     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3099
3100     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3101
3102         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3103         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3104     }
3105
3106     add_hyperv_enlighments($cpuFlags, $winversion, $machine_type, $kvmver, $nokvm, $conf->{bios}, $gpu_passthrough);
3107
3108     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3109
3110     push @$cpuFlags, 'kvm=off' if $kvm_off;
3111
3112     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3113         die "internal error"; # should not happen
3114
3115     push @$cpuFlags, "vendor=${cpu_vendor}"
3116         if $cpu_vendor ne 'default';
3117
3118     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3119
3120     push @$cmd, '-cpu', $cpu;
3121
3122     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3123
3124     push @$cmd, '-S' if $conf->{freeze};
3125
3126     # set keyboard layout
3127     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3128     push @$cmd, '-k', $kb if $kb;
3129
3130     # enable sound
3131     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3132     #push @$cmd, '-soundhw', 'es1370';
3133     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3134
3135     if($conf->{agent}) {
3136         my $qgasocket = qmp_socket($vmid, 1);
3137         my $pciaddr = print_pci_addr("qga0", $bridges);
3138         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3139         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3140         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3141     }
3142
3143     my $spice_port;
3144
3145     if ($qxlnum) {
3146         if ($qxlnum > 1) {
3147             if ($winversion){
3148                 for(my $i = 1; $i < $qxlnum; $i++){
3149                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3150                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3151                 }
3152             } else {
3153                 # assume other OS works like Linux
3154                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3155                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3156             }
3157         }
3158
3159         my $pciaddr = print_pci_addr("spice", $bridges);
3160
3161         my $nodename = PVE::INotify::nodename();
3162         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3163         $spice_port = PVE::Tools::next_spice_port($pfamily);
3164
3165         push @$devices, '-spice', "tls-port=${spice_port},addr=localhost,tls-ciphers=HIGH,seamless-migration=on";
3166
3167         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3168         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3169         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3170     }
3171
3172     # enable balloon by default, unless explicitly disabled
3173     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3174         $pciaddr = print_pci_addr("balloon0", $bridges);
3175         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3176     }
3177
3178     if ($conf->{watchdog}) {
3179         my $wdopts = parse_watchdog($conf->{watchdog});
3180         $pciaddr = print_pci_addr("watchdog", $bridges);
3181         my $watchdog = $wdopts->{model} || 'i6300esb';
3182         push @$devices, '-device', "$watchdog$pciaddr";
3183         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3184     }
3185
3186     my $vollist = [];
3187     my $scsicontroller = {};
3188     my $ahcicontroller = {};
3189     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3190
3191     # Add iscsi initiator name if available
3192     if (my $initiator = get_initiator_name()) {
3193         push @$devices, '-iscsi', "initiator-name=$initiator";
3194     }
3195
3196     foreach_drive($conf, sub {
3197         my ($ds, $drive) = @_;
3198
3199         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3200             push @$vollist, $drive->{file};
3201         }
3202
3203         $use_virtio = 1 if $ds =~ m/^virtio/;
3204
3205         if (drive_is_cdrom ($drive)) {
3206             if ($bootindex_hash->{d}) {
3207                 $drive->{bootindex} = $bootindex_hash->{d};
3208                 $bootindex_hash->{d} += 1;
3209             }
3210         } else {
3211             if ($bootindex_hash->{c}) {
3212                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3213                 $bootindex_hash->{c} += 1;
3214             }
3215         }
3216
3217         if($drive->{interface} eq 'virtio'){
3218            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3219         }
3220
3221         if ($drive->{interface} eq 'scsi') {
3222
3223             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3224
3225             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3226             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3227
3228             my $iothread = '';
3229             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3230                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3231                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3232             } elsif ($drive->{iothread}) {
3233                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3234             }
3235
3236             my $queues = '';
3237             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3238                 $queues = ",num_queues=$drive->{queues}";
3239             }
3240
3241             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3242             $scsicontroller->{$controller}=1;
3243         }
3244
3245         if ($drive->{interface} eq 'sata') {
3246            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3247            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3248            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3249            $ahcicontroller->{$controller}=1;
3250         }
3251
3252         if ($drive->{interface} eq 'efidisk') {
3253             # this will be added somewhere else
3254             return;
3255         }
3256
3257         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3258         push @$devices, '-drive',$drive_cmd;
3259         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3260     });
3261
3262     for (my $i = 0; $i < $MAX_NETS; $i++) {
3263          next if !$conf->{"net$i"};
3264          my $d = parse_net($conf->{"net$i"});
3265          next if !$d;
3266
3267          $use_virtio = 1 if $d->{model} eq 'virtio';
3268
3269          if ($bootindex_hash->{n}) {
3270             $d->{bootindex} = $bootindex_hash->{n};
3271             $bootindex_hash->{n} += 1;
3272          }
3273
3274          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3275          push @$devices, '-netdev', $netdevfull;
3276
3277          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3278          push @$devices, '-device', $netdevicefull;
3279     }
3280
3281     if (!$q35) {
3282         # add pci bridges
3283         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3284            $bridges->{1} = 1;
3285            $bridges->{2} = 1;
3286         }
3287
3288         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3289
3290         while (my ($k, $v) = each %$bridges) {
3291             $pciaddr = print_pci_addr("pci.$k");
3292             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3293         }
3294     }
3295
3296     # add custom args
3297     if ($conf->{args}) {
3298         my $aa = PVE::Tools::split_args($conf->{args});
3299         push @$cmd, @$aa;
3300     }
3301
3302     push @$cmd, @$devices;
3303     push @$cmd, '-rtc', join(',', @$rtcFlags)
3304         if scalar(@$rtcFlags);
3305     push @$cmd, '-machine', join(',', @$machineFlags)
3306         if scalar(@$machineFlags);
3307     push @$cmd, '-global', join(',', @$globalFlags)
3308         if scalar(@$globalFlags);
3309
3310     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3311 }
3312
3313 sub vnc_socket {
3314     my ($vmid) = @_;
3315     return "${var_run_tmpdir}/$vmid.vnc";
3316 }
3317
3318 sub spice_port {
3319     my ($vmid) = @_;
3320
3321     my $res = vm_mon_cmd($vmid, 'query-spice');
3322
3323     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3324 }
3325
3326 sub qmp_socket {
3327     my ($vmid, $qga) = @_;
3328     my $sockettype = $qga ? 'qga' : 'qmp';
3329     return "${var_run_tmpdir}/$vmid.$sockettype";
3330 }
3331
3332 sub pidfile_name {
3333     my ($vmid) = @_;
3334     return "${var_run_tmpdir}/$vmid.pid";
3335 }
3336
3337 sub vm_devices_list {
3338     my ($vmid) = @_;
3339
3340     my $res = vm_mon_cmd($vmid, 'query-pci');
3341     my $devices = {};
3342     foreach my $pcibus (@$res) {
3343         foreach my $device (@{$pcibus->{devices}}) {
3344             next if !$device->{'qdev_id'};
3345             if ($device->{'pci_bridge'}) {
3346                 $devices->{$device->{'qdev_id'}} = 1;
3347                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3348                     next if !$bridge_device->{'qdev_id'};
3349                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3350                     $devices->{$device->{'qdev_id'}}++;
3351                 }
3352             } else {
3353                 $devices->{$device->{'qdev_id'}} = 1;
3354             }
3355         }
3356     }
3357
3358     my $resblock = vm_mon_cmd($vmid, 'query-block');
3359     foreach my $block (@$resblock) {
3360         if($block->{device} =~ m/^drive-(\S+)/){
3361                 $devices->{$1} = 1;
3362         }
3363     }
3364
3365     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3366     foreach my $mice (@$resmice) {
3367         if ($mice->{name} eq 'QEMU HID Tablet') {
3368             $devices->{tablet} = 1;
3369             last;
3370         }
3371     }
3372
3373     # for usb devices there is no query-usb
3374     # but we can iterate over the entries in
3375     # qom-list path=/machine/peripheral
3376     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3377     foreach my $per (@$resperipheral) {
3378         if ($per->{name} =~ m/^usb\d+$/) {
3379             $devices->{$per->{name}} = 1;
3380         }
3381     }
3382
3383     return $devices;
3384 }
3385
3386 sub vm_deviceplug {
3387     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3388
3389     my $q35 = machine_type_is_q35($conf);
3390
3391     my $devices_list = vm_devices_list($vmid);
3392     return 1 if defined($devices_list->{$deviceid});
3393
3394     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3395
3396     if ($deviceid eq 'tablet') {
3397
3398         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3399
3400     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3401
3402         die "usb hotplug currently not reliable\n";
3403         # since we can't reliably hot unplug all added usb devices
3404         # and usb passthrough disables live migration
3405         # we disable usb hotplugging for now
3406         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3407
3408     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3409
3410         qemu_iothread_add($vmid, $deviceid, $device);
3411
3412         qemu_driveadd($storecfg, $vmid, $device);
3413         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3414
3415         qemu_deviceadd($vmid, $devicefull);
3416         eval { qemu_deviceaddverify($vmid, $deviceid); };
3417         if (my $err = $@) {
3418             eval { qemu_drivedel($vmid, $deviceid); };
3419             warn $@ if $@;
3420             die $err;
3421         }
3422
3423     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3424
3425
3426         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3427         my $pciaddr = print_pci_addr($deviceid);
3428         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3429
3430         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3431
3432         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3433             qemu_iothread_add($vmid, $deviceid, $device);
3434             $devicefull .= ",iothread=iothread-$deviceid";
3435         }
3436
3437         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3438             $devicefull .= ",num_queues=$device->{queues}";
3439         }
3440
3441         qemu_deviceadd($vmid, $devicefull);
3442         qemu_deviceaddverify($vmid, $deviceid);
3443
3444     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3445
3446         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3447         qemu_driveadd($storecfg, $vmid, $device);
3448
3449         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3450         eval { qemu_deviceadd($vmid, $devicefull); };
3451         if (my $err = $@) {
3452             eval { qemu_drivedel($vmid, $deviceid); };
3453             warn $@ if $@;
3454             die $err;
3455         }
3456
3457     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3458
3459         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3460
3461         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3462         my $use_old_bios_files = undef;
3463         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3464
3465         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3466         qemu_deviceadd($vmid, $netdevicefull);
3467         eval { qemu_deviceaddverify($vmid, $deviceid); };
3468         if (my $err = $@) {
3469             eval { qemu_netdevdel($vmid, $deviceid); };
3470             warn $@ if $@;
3471             die $err;
3472         }
3473
3474     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3475
3476         my $bridgeid = $2;
3477         my $pciaddr = print_pci_addr($deviceid);
3478         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3479
3480         qemu_deviceadd($vmid, $devicefull);
3481         qemu_deviceaddverify($vmid, $deviceid);
3482
3483     } else {
3484         die "can't hotplug device '$deviceid'\n";
3485     }
3486
3487     return 1;
3488 }
3489
3490 # fixme: this should raise exceptions on error!
3491 sub vm_deviceunplug {
3492     my ($vmid, $conf, $deviceid) = @_;
3493
3494     my $devices_list = vm_devices_list($vmid);
3495     return 1 if !defined($devices_list->{$deviceid});
3496
3497     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3498
3499     if ($deviceid eq 'tablet') {
3500
3501         qemu_devicedel($vmid, $deviceid);
3502
3503     } elsif ($deviceid =~ m/^usb\d+$/) {
3504
3505         die "usb hotplug currently not reliable\n";
3506         # when unplugging usb devices this way,
3507         # there may be remaining usb controllers/hubs
3508         # so we disable it for now
3509         qemu_devicedel($vmid, $deviceid);
3510         qemu_devicedelverify($vmid, $deviceid);
3511
3512     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3513
3514         qemu_devicedel($vmid, $deviceid);
3515         qemu_devicedelverify($vmid, $deviceid);
3516         qemu_drivedel($vmid, $deviceid);
3517         qemu_iothread_del($conf, $vmid, $deviceid);
3518
3519     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3520
3521         qemu_devicedel($vmid, $deviceid);
3522         qemu_devicedelverify($vmid, $deviceid);
3523         qemu_iothread_del($conf, $vmid, $deviceid);
3524
3525     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3526
3527         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3528         my $device = parse_drive($deviceid, $conf->{$deviceid});
3529         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3530
3531         qemu_devicedel($vmid, $deviceid);
3532         qemu_drivedel($vmid, $deviceid);
3533         qemu_deletescsihw($conf, $vmid, $deviceid);
3534
3535     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3536
3537         qemu_devicedel($vmid, $deviceid);
3538         qemu_devicedelverify($vmid, $deviceid);
3539         qemu_netdevdel($vmid, $deviceid);
3540
3541     } else {
3542         die "can't unplug device '$deviceid'\n";
3543     }
3544
3545     return 1;
3546 }
3547
3548 sub qemu_deviceadd {
3549     my ($vmid, $devicefull) = @_;
3550
3551     $devicefull = "driver=".$devicefull;
3552     my %options =  split(/[=,]/, $devicefull);
3553
3554     vm_mon_cmd($vmid, "device_add" , %options);
3555 }
3556
3557 sub qemu_devicedel {
3558     my ($vmid, $deviceid) = @_;
3559
3560     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3561 }
3562
3563 sub qemu_iothread_add {
3564     my($vmid, $deviceid, $device) = @_;
3565
3566     if ($device->{iothread}) {
3567         my $iothreads = vm_iothreads_list($vmid);
3568         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3569     }
3570 }
3571
3572 sub qemu_iothread_del {
3573     my($conf, $vmid, $deviceid) = @_;
3574
3575     my $device = parse_drive($deviceid, $conf->{$deviceid});
3576     if ($device->{iothread}) {
3577         my $iothreads = vm_iothreads_list($vmid);
3578         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3579     }
3580 }
3581
3582 sub qemu_objectadd {
3583     my($vmid, $objectid, $qomtype) = @_;
3584
3585     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3586
3587     return 1;
3588 }
3589
3590 sub qemu_objectdel {
3591     my($vmid, $objectid) = @_;
3592
3593     vm_mon_cmd($vmid, "object-del", id => $objectid);
3594
3595     return 1;
3596 }
3597
3598 sub qemu_driveadd {
3599     my ($storecfg, $vmid, $device) = @_;
3600
3601     my $drive = print_drive_full($storecfg, $vmid, $device);
3602     $drive =~ s/\\/\\\\/g;
3603     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3604
3605     # If the command succeeds qemu prints: "OK"
3606     return 1 if $ret =~ m/OK/s;
3607
3608     die "adding drive failed: $ret\n";
3609 }
3610
3611 sub qemu_drivedel {
3612     my($vmid, $deviceid) = @_;
3613
3614     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3615     $ret =~ s/^\s+//;
3616
3617     return 1 if $ret eq "";
3618
3619     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3620     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3621
3622     die "deleting drive $deviceid failed : $ret\n";
3623 }
3624
3625 sub qemu_deviceaddverify {
3626     my ($vmid, $deviceid) = @_;
3627
3628     for (my $i = 0; $i <= 5; $i++) {
3629          my $devices_list = vm_devices_list($vmid);
3630          return 1 if defined($devices_list->{$deviceid});
3631          sleep 1;
3632     }
3633
3634     die "error on hotplug device '$deviceid'\n";
3635 }
3636
3637
3638 sub qemu_devicedelverify {
3639     my ($vmid, $deviceid) = @_;
3640
3641     # need to verify that the device is correctly removed as device_del
3642     # is async and empty return is not reliable
3643
3644     for (my $i = 0; $i <= 5; $i++) {
3645          my $devices_list = vm_devices_list($vmid);
3646          return 1 if !defined($devices_list->{$deviceid});
3647          sleep 1;
3648     }
3649
3650     die "error on hot-unplugging device '$deviceid'\n";
3651 }
3652
3653 sub qemu_findorcreatescsihw {
3654     my ($storecfg, $conf, $vmid, $device) = @_;
3655
3656     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3657
3658     my $scsihwid="$controller_prefix$controller";
3659     my $devices_list = vm_devices_list($vmid);
3660
3661     if(!defined($devices_list->{$scsihwid})) {
3662         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3663     }
3664
3665     return 1;
3666 }
3667
3668 sub qemu_deletescsihw {
3669     my ($conf, $vmid, $opt) = @_;
3670
3671     my $device = parse_drive($opt, $conf->{$opt});
3672
3673     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3674         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3675         return 1;
3676     }
3677
3678     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3679
3680     my $devices_list = vm_devices_list($vmid);
3681     foreach my $opt (keys %{$devices_list}) {
3682         if (PVE::QemuServer::is_valid_drivename($opt)) {
3683             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3684             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3685                 return 1;
3686             }
3687         }
3688     }
3689
3690     my $scsihwid="scsihw$controller";
3691
3692     vm_deviceunplug($vmid, $conf, $scsihwid);
3693
3694     return 1;
3695 }
3696
3697 sub qemu_add_pci_bridge {
3698     my ($storecfg, $conf, $vmid, $device) = @_;
3699
3700     my $bridges = {};
3701
3702     my $bridgeid;
3703
3704     print_pci_addr($device, $bridges);
3705
3706     while (my ($k, $v) = each %$bridges) {
3707         $bridgeid = $k;
3708     }
3709     return 1 if !defined($bridgeid) || $bridgeid < 1;
3710
3711     my $bridge = "pci.$bridgeid";
3712     my $devices_list = vm_devices_list($vmid);
3713
3714     if (!defined($devices_list->{$bridge})) {
3715         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3716     }
3717
3718     return 1;
3719 }
3720
3721 sub qemu_set_link_status {
3722     my ($vmid, $device, $up) = @_;
3723
3724     vm_mon_cmd($vmid, "set_link", name => $device,
3725                up => $up ? JSON::true : JSON::false);
3726 }
3727
3728 sub qemu_netdevadd {
3729     my ($vmid, $conf, $device, $deviceid) = @_;
3730
3731     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3732     my %options =  split(/[=,]/, $netdev);
3733
3734     vm_mon_cmd($vmid, "netdev_add",  %options);
3735     return 1;
3736 }
3737
3738 sub qemu_netdevdel {
3739     my ($vmid, $deviceid) = @_;
3740
3741     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3742 }
3743
3744 sub qemu_usb_hotplug {
3745     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3746
3747     return if !$device;
3748
3749     # remove the old one first
3750     vm_deviceunplug($vmid, $conf, $deviceid);
3751
3752     # check if xhci controller is necessary and available
3753     if ($device->{usb3}) {
3754
3755         my $devicelist = vm_devices_list($vmid);
3756
3757         if (!$devicelist->{xhci}) {
3758             my $pciaddr = print_pci_addr("xhci");
3759             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3760         }
3761     }
3762     my $d = parse_usb_device($device->{host});
3763     $d->{usb3} = $device->{usb3};
3764
3765     # add the new one
3766     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3767 }
3768
3769 sub qemu_cpu_hotplug {
3770     my ($vmid, $conf, $vcpus) = @_;
3771
3772     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3773
3774     my $sockets = 1;
3775     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3776     $sockets = $conf->{sockets} if  $conf->{sockets};
3777     my $cores = $conf->{cores} || 1;
3778     my $maxcpus = $sockets * $cores;
3779
3780     $vcpus = $maxcpus if !$vcpus;
3781
3782     die "you can't add more vcpus than maxcpus\n"
3783         if $vcpus > $maxcpus;
3784
3785     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3786
3787     if ($vcpus < $currentvcpus) {
3788
3789         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3790
3791             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3792                 qemu_devicedel($vmid, "cpu$i");
3793                 my $retry = 0;
3794                 my $currentrunningvcpus = undef;
3795                 while (1) {
3796                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3797                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3798                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3799                     $retry++;
3800                     sleep 1;
3801                 }
3802                 #update conf after each succesfull cpu unplug
3803                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3804                 PVE::QemuConfig->write_config($vmid, $conf);
3805             }
3806         } else {
3807             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3808         }
3809
3810         return;
3811     }
3812
3813     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3814     die "vcpus in running vm does not match its configuration\n"
3815         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3816
3817     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3818
3819         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3820             my $cpustr = print_cpu_device($conf, $i);
3821             qemu_deviceadd($vmid, $cpustr);
3822
3823             my $retry = 0;
3824             my $currentrunningvcpus = undef;
3825             while (1) {
3826                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3827                 last if scalar(@{$currentrunningvcpus}) == $i;
3828                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3829                 sleep 1;
3830                 $retry++;
3831             }
3832             #update conf after each succesfull cpu hotplug
3833             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3834             PVE::QemuConfig->write_config($vmid, $conf);
3835         }
3836     } else {
3837
3838         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3839             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3840         }
3841     }
3842 }
3843
3844 sub qemu_block_set_io_throttle {
3845     my ($vmid, $deviceid,
3846         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3847         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3848         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3849         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3850
3851     return if !check_running($vmid) ;
3852
3853     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3854         bps => int($bps),
3855         bps_rd => int($bps_rd),
3856         bps_wr => int($bps_wr),
3857         iops => int($iops),
3858         iops_rd => int($iops_rd),
3859         iops_wr => int($iops_wr),
3860         bps_max => int($bps_max),
3861         bps_rd_max => int($bps_rd_max),
3862         bps_wr_max => int($bps_wr_max),
3863         iops_max => int($iops_max),
3864         iops_rd_max => int($iops_rd_max),
3865         iops_wr_max => int($iops_wr_max),
3866         bps_max_length => int($bps_max_length),
3867         bps_rd_max_length => int($bps_rd_max_length),
3868         bps_wr_max_length => int($bps_wr_max_length),
3869         iops_max_length => int($iops_max_length),
3870         iops_rd_max_length => int($iops_rd_max_length),
3871         iops_wr_max_length => int($iops_wr_max_length),
3872     );
3873
3874 }
3875
3876 # old code, only used to shutdown old VM after update
3877 sub __read_avail {
3878     my ($fh, $timeout) = @_;
3879
3880     my $sel = new IO::Select;
3881     $sel->add($fh);
3882
3883     my $res = '';
3884     my $buf;
3885
3886     my @ready;
3887     while (scalar (@ready = $sel->can_read($timeout))) {
3888         my $count;
3889         if ($count = $fh->sysread($buf, 8192)) {
3890             if ($buf =~ /^(.*)\(qemu\) $/s) {
3891                 $res .= $1;
3892                 last;
3893             } else {
3894                 $res .= $buf;
3895             }
3896         } else {
3897             if (!defined($count)) {
3898                 die "$!\n";
3899             }
3900             last;
3901         }
3902     }
3903
3904     die "monitor read timeout\n" if !scalar(@ready);
3905
3906     return $res;
3907 }
3908
3909 # old code, only used to shutdown old VM after update
3910 sub vm_monitor_command {
3911     my ($vmid, $cmdstr, $nocheck) = @_;
3912
3913     my $res;
3914
3915     eval {
3916         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3917
3918         my $sname = "${var_run_tmpdir}/$vmid.mon";
3919
3920         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3921             die "unable to connect to VM $vmid socket - $!\n";
3922
3923         my $timeout = 3;
3924
3925         # hack: migrate sometime blocks the monitor (when migrate_downtime
3926         # is set)
3927         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3928             $timeout = 60*60; # 1 hour
3929         }
3930
3931         # read banner;
3932         my $data = __read_avail($sock, $timeout);
3933
3934         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3935             die "got unexpected qemu monitor banner\n";
3936         }
3937
3938         my $sel = new IO::Select;
3939         $sel->add($sock);
3940
3941         if (!scalar(my @ready = $sel->can_write($timeout))) {
3942             die "monitor write error - timeout";
3943         }
3944
3945         my $fullcmd = "$cmdstr\r";
3946
3947         # syslog('info', "VM $vmid monitor command: $cmdstr");
3948
3949         my $b;
3950         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3951             die "monitor write error - $!";
3952         }
3953
3954         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3955
3956         $timeout = 20;
3957
3958         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3959             $timeout = 60*60; # 1 hour
3960         } elsif ($cmdstr =~ m/^(eject|change)/) {
3961             $timeout = 60; # note: cdrom mount command is slow
3962         }
3963         if ($res = __read_avail($sock, $timeout)) {
3964
3965             my @lines = split("\r?\n", $res);
3966
3967             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3968
3969             $res = join("\n", @lines);
3970             $res .= "\n";
3971         }
3972     };
3973
3974     my $err = $@;
3975
3976     if ($err) {
3977         syslog("err", "VM $vmid monitor command failed - $err");
3978         die $err;
3979     }
3980
3981     return $res;
3982 }
3983
3984 sub qemu_block_resize {
3985     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3986
3987     my $running = check_running($vmid);
3988
3989     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3990
3991     return if !$running;
3992
3993     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3994
3995 }
3996
3997 sub qemu_volume_snapshot {
3998     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3999
4000     my $running = check_running($vmid);
4001
4002     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4003         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4004     } else {
4005         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4006     }
4007 }
4008
4009 sub qemu_volume_snapshot_delete {
4010     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4011
4012     my $running = check_running($vmid);
4013
4014     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4015         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4016     } else {
4017         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4018     }
4019 }
4020
4021 sub set_migration_caps {
4022     my ($vmid) = @_;
4023
4024     my $cap_ref = [];
4025
4026     my $enabled_cap = {
4027         "auto-converge" => 1,
4028         "xbzrle" => 1,
4029         "x-rdma-pin-all" => 0,
4030         "zero-blocks" => 0,
4031         "compress" => 0
4032     };
4033
4034     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4035
4036     for my $supported_capability (@$supported_capabilities) {
4037         push @$cap_ref, {
4038             capability => $supported_capability->{capability},
4039             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4040         };
4041     }
4042
4043     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4044 }
4045
4046 my $fast_plug_option = {
4047     'lock' => 1,
4048     'name' => 1,
4049     'onboot' => 1,
4050     'shares' => 1,
4051     'startup' => 1,
4052     'description' => 1,
4053     'protection' => 1,
4054 };
4055
4056 # hotplug changes in [PENDING]
4057 # $selection hash can be used to only apply specified options, for
4058 # example: { cores => 1 } (only apply changed 'cores')
4059 # $errors ref is used to return error messages
4060 sub vmconfig_hotplug_pending {
4061     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4062
4063     my $defaults = load_defaults();
4064
4065     # commit values which do not have any impact on running VM first
4066     # Note: those option cannot raise errors, we we do not care about
4067     # $selection and always apply them.
4068
4069     my $add_error = sub {
4070         my ($opt, $msg) = @_;
4071         $errors->{$opt} = "hotplug problem - $msg";
4072     };
4073
4074     my $changes = 0;
4075     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4076         if ($fast_plug_option->{$opt}) {
4077             $conf->{$opt} = $conf->{pending}->{$opt};
4078             delete $conf->{pending}->{$opt};
4079             $changes = 1;
4080         }
4081     }
4082
4083     if ($changes) {
4084         PVE::QemuConfig->write_config($vmid, $conf);
4085         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4086     }
4087
4088     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4089
4090     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4091     while (my ($opt, $force) = each %$pending_delete_hash) {
4092         next if $selection && !$selection->{$opt};
4093         eval {
4094             if ($opt eq 'hotplug') {
4095                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4096             } elsif ($opt eq 'tablet') {
4097                 die "skip\n" if !$hotplug_features->{usb};
4098                 if ($defaults->{tablet}) {
4099                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4100                 } else {
4101                     vm_deviceunplug($vmid, $conf, $opt);
4102                 }
4103             } elsif ($opt =~ m/^usb\d+/) {
4104                 die "skip\n";
4105                 # since we cannot reliably hot unplug usb devices
4106                 # we are disabling it
4107                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4108                 vm_deviceunplug($vmid, $conf, $opt);
4109             } elsif ($opt eq 'vcpus') {
4110                 die "skip\n" if !$hotplug_features->{cpu};
4111                 qemu_cpu_hotplug($vmid, $conf, undef);
4112             } elsif ($opt eq 'balloon') {
4113                 # enable balloon device is not hotpluggable
4114                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4115             } elsif ($fast_plug_option->{$opt}) {
4116                 # do nothing
4117             } elsif ($opt =~ m/^net(\d+)$/) {
4118                 die "skip\n" if !$hotplug_features->{network};
4119                 vm_deviceunplug($vmid, $conf, $opt);
4120             } elsif (is_valid_drivename($opt)) {
4121                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4122                 vm_deviceunplug($vmid, $conf, $opt);
4123                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4124             } elsif ($opt =~ m/^memory$/) {
4125                 die "skip\n" if !$hotplug_features->{memory};
4126                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4127             } elsif ($opt eq 'cpuunits') {
4128                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4129             } elsif ($opt eq 'cpulimit') {
4130                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4131             } else {
4132                 die "skip\n";
4133             }
4134         };
4135         if (my $err = $@) {
4136             &$add_error($opt, $err) if $err ne "skip\n";
4137         } else {
4138             # save new config if hotplug was successful
4139             delete $conf->{$opt};
4140             vmconfig_undelete_pending_option($conf, $opt);
4141             PVE::QemuConfig->write_config($vmid, $conf);
4142             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4143         }
4144     }
4145
4146     foreach my $opt (keys %{$conf->{pending}}) {
4147         next if $selection && !$selection->{$opt};
4148         my $value = $conf->{pending}->{$opt};
4149         eval {
4150             if ($opt eq 'hotplug') {
4151                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4152             } elsif ($opt eq 'tablet') {
4153                 die "skip\n" if !$hotplug_features->{usb};
4154                 if ($value == 1) {
4155                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4156                 } elsif ($value == 0) {
4157                     vm_deviceunplug($vmid, $conf, $opt);
4158                 }
4159             } elsif ($opt =~ m/^usb\d+$/) {
4160                 die "skip\n";
4161                 # since we cannot reliably hot unplug usb devices
4162                 # we are disabling it
4163                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4164                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4165                 die "skip\n" if !$d;
4166                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4167             } elsif ($opt eq 'vcpus') {
4168                 die "skip\n" if !$hotplug_features->{cpu};
4169                 qemu_cpu_hotplug($vmid, $conf, $value);
4170             } elsif ($opt eq 'balloon') {
4171                 # enable/disable balloning device is not hotpluggable
4172                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4173                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4174                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4175
4176                 # allow manual ballooning if shares is set to zero
4177                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4178                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4179                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4180                 }
4181             } elsif ($opt =~ m/^net(\d+)$/) {
4182                 # some changes can be done without hotplug
4183                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4184                                     $vmid, $opt, $value);
4185             } elsif (is_valid_drivename($opt)) {
4186                 # some changes can be done without hotplug
4187                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4188                                      $vmid, $opt, $value, 1);
4189             } elsif ($opt =~ m/^memory$/) { #dimms
4190                 die "skip\n" if !$hotplug_features->{memory};
4191                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4192             } elsif ($opt eq 'cpuunits') {
4193                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4194             } elsif ($opt eq 'cpulimit') {
4195                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4196                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4197             } else {
4198                 die "skip\n";  # skip non-hot-pluggable options
4199             }
4200         };
4201         if (my $err = $@) {
4202             &$add_error($opt, $err) if $err ne "skip\n";
4203         } else {
4204             # save new config if hotplug was successful
4205             $conf->{$opt} = $value;
4206             delete $conf->{pending}->{$opt};
4207             PVE::QemuConfig->write_config($vmid, $conf);
4208             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4209         }
4210     }
4211 }
4212
4213 sub try_deallocate_drive {
4214     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4215
4216     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4217         my $volid = $drive->{file};
4218         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4219             my $sid = PVE::Storage::parse_volume_id($volid);
4220             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4221
4222             # check if the disk is really unused
4223             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4224                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4225             PVE::Storage::vdisk_free($storecfg, $volid);
4226             return 1;
4227         } else {
4228             # If vm is not owner of this disk remove from config
4229             return 1;
4230         }
4231     }
4232
4233     return undef;
4234 }
4235
4236 sub vmconfig_delete_or_detach_drive {
4237     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4238
4239     my $drive = parse_drive($opt, $conf->{$opt});
4240
4241     my $rpcenv = PVE::RPCEnvironment::get();
4242     my $authuser = $rpcenv->get_user();
4243
4244     if ($force) {
4245         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4246         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4247     } else {
4248         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4249     }
4250 }
4251
4252 sub vmconfig_apply_pending {
4253     my ($vmid, $conf, $storecfg) = @_;
4254
4255     # cold plug
4256
4257     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4258     while (my ($opt, $force) = each %$pending_delete_hash) {
4259         die "internal error" if $opt =~ m/^unused/;
4260         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4261         if (!defined($conf->{$opt})) {
4262             vmconfig_undelete_pending_option($conf, $opt);
4263             PVE::QemuConfig->write_config($vmid, $conf);
4264         } elsif (is_valid_drivename($opt)) {
4265             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4266             vmconfig_undelete_pending_option($conf, $opt);
4267             delete $conf->{$opt};
4268             PVE::QemuConfig->write_config($vmid, $conf);
4269         } else {
4270             vmconfig_undelete_pending_option($conf, $opt);
4271             delete $conf->{$opt};
4272             PVE::QemuConfig->write_config($vmid, $conf);
4273         }
4274     }
4275
4276     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4277
4278     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4279         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4280
4281         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4282             # skip if nothing changed
4283         } elsif (is_valid_drivename($opt)) {
4284             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4285                 if defined($conf->{$opt});
4286             $conf->{$opt} = $conf->{pending}->{$opt};
4287         } else {
4288             $conf->{$opt} = $conf->{pending}->{$opt};
4289         }
4290
4291         delete $conf->{pending}->{$opt};
4292         PVE::QemuConfig->write_config($vmid, $conf);
4293     }
4294 }
4295
4296 my $safe_num_ne = sub {
4297     my ($a, $b) = @_;
4298
4299     return 0 if !defined($a) && !defined($b);
4300     return 1 if !defined($a);
4301     return 1 if !defined($b);
4302
4303     return $a != $b;
4304 };
4305
4306 my $safe_string_ne = sub {
4307     my ($a, $b) = @_;
4308
4309     return 0 if !defined($a) && !defined($b);
4310     return 1 if !defined($a);
4311     return 1 if !defined($b);
4312
4313     return $a ne $b;
4314 };
4315
4316 sub vmconfig_update_net {
4317     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4318
4319     my $newnet = parse_net($value);
4320
4321     if ($conf->{$opt}) {
4322         my $oldnet = parse_net($conf->{$opt});
4323
4324         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4325             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4326             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4327             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4328
4329             # for non online change, we try to hot-unplug
4330             die "skip\n" if !$hotplug;
4331             vm_deviceunplug($vmid, $conf, $opt);
4332         } else {
4333
4334             die "internal error" if $opt !~ m/net(\d+)/;
4335             my $iface = "tap${vmid}i$1";
4336
4337             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4338                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4339                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4340                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4341                 PVE::Network::tap_unplug($iface);
4342                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4343             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4344                 # Rate can be applied on its own but any change above needs to
4345                 # include the rate in tap_plug since OVS resets everything.
4346                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4347             }
4348
4349             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4350                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4351             }
4352
4353             return 1;
4354         }
4355     }
4356
4357     if ($hotplug) {
4358         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4359     } else {
4360         die "skip\n";
4361     }
4362 }
4363
4364 sub vmconfig_update_disk {
4365     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4366
4367     # fixme: do we need force?
4368
4369     my $drive = parse_drive($opt, $value);
4370
4371     if ($conf->{$opt}) {
4372
4373         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4374
4375             my $media = $drive->{media} || 'disk';
4376             my $oldmedia = $old_drive->{media} || 'disk';
4377             die "unable to change media type\n" if $media ne $oldmedia;
4378
4379             if (!drive_is_cdrom($old_drive)) {
4380
4381                 if ($drive->{file} ne $old_drive->{file}) {
4382
4383                     die "skip\n" if !$hotplug;
4384
4385                     # unplug and register as unused
4386                     vm_deviceunplug($vmid, $conf, $opt);
4387                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4388
4389                 } else {
4390                     # update existing disk
4391
4392                     # skip non hotpluggable value
4393                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4394                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4395                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4396                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4397                         die "skip\n";
4398                     }
4399
4400                     # apply throttle
4401                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4402                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4403                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4404                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4405                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4406                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4407                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4408                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4409                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4410                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4411                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4412                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4413                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4414                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4415                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4416                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4417                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4418                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4419
4420                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4421                                                    ($drive->{mbps} || 0)*1024*1024,
4422                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4423                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4424                                                    $drive->{iops} || 0,
4425                                                    $drive->{iops_rd} || 0,
4426                                                    $drive->{iops_wr} || 0,
4427                                                    ($drive->{mbps_max} || 0)*1024*1024,
4428                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4429                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4430                                                    $drive->{iops_max} || 0,
4431                                                    $drive->{iops_rd_max} || 0,
4432                                                    $drive->{iops_wr_max} || 0,
4433                                                    $drive->{bps_max_length} || 1,
4434                                                    $drive->{bps_rd_max_length} || 1,
4435                                                    $drive->{bps_wr_max_length} || 1,
4436                                                    $drive->{iops_max_length} || 1,
4437                                                    $drive->{iops_rd_max_length} || 1,
4438                                                    $drive->{iops_wr_max_length} || 1);
4439
4440                     }
4441
4442                     return 1;
4443                 }
4444
4445             } else { # cdrom
4446
4447                 if ($drive->{file} eq 'none') {
4448                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4449                 } else {
4450                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4451                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4452                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4453                 }
4454
4455                 return 1;
4456             }
4457         }
4458     }
4459
4460     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4461     # hotplug new disks
4462     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4463     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4464 }
4465
4466 sub vm_start {
4467     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4468         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4469
4470     PVE::QemuConfig->lock_config($vmid, sub {
4471         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4472
4473         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4474
4475         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4476
4477         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4478
4479         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4480             vmconfig_apply_pending($vmid, $conf, $storecfg);
4481             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4482         }
4483
4484         my $defaults = load_defaults();
4485
4486         # set environment variable useful inside network script
4487         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4488
4489         my $local_volumes = {};
4490
4491         if ($targetstorage) {
4492             foreach_drive($conf, sub {
4493                 my ($ds, $drive) = @_;
4494
4495                 return if drive_is_cdrom($drive);
4496
4497                 my $volid = $drive->{file};
4498
4499                 return if !$volid;
4500
4501                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4502
4503                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4504                 return if $scfg->{shared};
4505                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4506             });
4507
4508             my $format = undef;
4509
4510             foreach my $opt (sort keys %$local_volumes) {
4511
4512                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4513                 my $drive = parse_drive($opt, $conf->{$opt});
4514
4515                 #if remote storage is specified, use default format
4516                 if ($targetstorage && $targetstorage ne "1") {
4517                     $storeid = $targetstorage;
4518                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4519                     $format = $defFormat;
4520                 } else {
4521                     #else we use same format than original
4522                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4523                     $format = qemu_img_format($scfg, $volid);
4524                 }
4525
4526                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4527                 my $newdrive = $drive;
4528                 $newdrive->{format} = $format;
4529                 $newdrive->{file} = $newvolid;
4530                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4531                 $local_volumes->{$opt} = $drivestr;
4532                 #pass drive to conf for command line
4533                 $conf->{$opt} = $drivestr;
4534             }
4535         }
4536
4537         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4538
4539         my $migrate_port = 0;
4540         my $migrate_uri;
4541         if ($statefile) {
4542             if ($statefile eq 'tcp') {
4543                 my $localip = "localhost";
4544                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4545                 my $nodename = PVE::INotify::nodename();
4546
4547                 if (!defined($migration_type)) {
4548                     if (defined($datacenterconf->{migration}->{type})) {
4549                         $migration_type = $datacenterconf->{migration}->{type};
4550                     } else {
4551                         $migration_type = 'secure';
4552                     }
4553                 }
4554
4555                 if ($migration_type eq 'insecure') {
4556                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4557                     if ($migrate_network_addr) {
4558                         $localip = $migrate_network_addr;
4559                     } else {
4560                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4561                     }
4562
4563                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4564                 }
4565
4566                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4567                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4568                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4569                 push @$cmd, '-incoming', $migrate_uri;
4570                 push @$cmd, '-S';
4571
4572             } elsif ($statefile eq 'unix') {
4573                 # should be default for secure migrations as a ssh TCP forward
4574                 # tunnel is not deterministic reliable ready and fails regurarly
4575                 # to set up in time, so use UNIX socket forwards
4576                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4577                 unlink $socket_addr;
4578
4579                 $migrate_uri = "unix:$socket_addr";
4580
4581                 push @$cmd, '-incoming', $migrate_uri;
4582                 push @$cmd, '-S';
4583
4584             } else {
4585                 push @$cmd, '-loadstate', $statefile;
4586             }
4587         } elsif ($paused) {
4588             push @$cmd, '-S';
4589         }
4590
4591         # host pci devices
4592         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4593           my $d = parse_hostpci($conf->{"hostpci$i"});
4594           next if !$d;
4595           my $pcidevices = $d->{pciid};
4596           foreach my $pcidevice (@$pcidevices) {
4597                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4598
4599                 my $info = pci_device_info("0000:$pciid");
4600                 die "IOMMU not present\n" if !check_iommu_support();
4601                 die "no pci device info for device '$pciid'\n" if !$info;
4602                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4603                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4604           }
4605         }
4606
4607         PVE::Storage::activate_volumes($storecfg, $vollist);
4608
4609         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4610             my $cmd = [];
4611             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4612             eval  { run_command($cmd); };
4613         }
4614
4615         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4616                                                   : $defaults->{cpuunits};
4617
4618         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4619
4620         my %properties = (
4621             Slice => 'qemu.slice',
4622             KillMode => 'none',
4623             CPUShares => $cpuunits
4624         );
4625
4626         if (my $cpulimit = $conf->{cpulimit}) {
4627             $properties{CPUQuota} = int($cpulimit * 100);
4628         }
4629         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4630
4631         if ($conf->{hugepages}) {
4632
4633             my $code = sub {
4634                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4635                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4636
4637                 PVE::QemuServer::Memory::hugepages_mount();
4638                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4639
4640                 eval  {
4641                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4642                     run_command($cmd, %run_params);
4643                 };
4644
4645                 if (my $err = $@) {
4646                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4647                     die $err;
4648                 }
4649
4650                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4651             };
4652             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4653
4654         } else {
4655             eval  {
4656                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4657                 run_command($cmd, %run_params);
4658             };
4659         }
4660
4661         if (my $err = $@) {
4662             # deactivate volumes if start fails
4663             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4664             die "start failed: $err";
4665         }
4666
4667         print "migration listens on $migrate_uri\n" if $migrate_uri;
4668
4669         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4670             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4671             warn $@ if $@;
4672         }
4673
4674         #start nbd server for storage migration
4675         if ($targetstorage) {
4676             my $nodename = PVE::INotify::nodename();
4677             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4678             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4679             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4680             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4681
4682             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4683
4684             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4685
4686             foreach my $opt (sort keys %$local_volumes) {
4687                 my $volid = $local_volumes->{$opt};
4688                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4689                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4690                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4691             }
4692         }
4693
4694         if ($migratedfrom) {
4695             eval {
4696                 set_migration_caps($vmid);
4697             };
4698             warn $@ if $@;
4699
4700             if ($spice_port) {
4701                 print "spice listens on port $spice_port\n";
4702                 if ($spice_ticket) {
4703                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4704                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4705                 }
4706             }
4707
4708         } else {
4709             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4710                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4711                     if $conf->{balloon};
4712             }
4713
4714             foreach my $opt (keys %$conf) {
4715                 next if $opt !~  m/^net\d+$/;
4716                 my $nicconf = parse_net($conf->{$opt});
4717                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4718             }
4719         }
4720
4721         vm_mon_cmd_nocheck($vmid, 'qom-set',
4722                     path => "machine/peripheral/balloon0",
4723                     property => "guest-stats-polling-interval",
4724                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4725
4726     });
4727 }
4728
4729 sub vm_mon_cmd {
4730     my ($vmid, $execute, %params) = @_;
4731
4732     my $cmd = { execute => $execute, arguments => \%params };
4733     vm_qmp_command($vmid, $cmd);
4734 }
4735
4736 sub vm_mon_cmd_nocheck {
4737     my ($vmid, $execute, %params) = @_;
4738
4739     my $cmd = { execute => $execute, arguments => \%params };
4740     vm_qmp_command($vmid, $cmd, 1);
4741 }
4742
4743 sub vm_qmp_command {
4744     my ($vmid, $cmd, $nocheck) = @_;
4745
4746     my $res;
4747
4748     my $timeout;
4749     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4750         $timeout = $cmd->{arguments}->{timeout};
4751         delete $cmd->{arguments}->{timeout};
4752     }
4753
4754     eval {
4755         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4756         my $sname = qmp_socket($vmid);
4757         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4758             my $qmpclient = PVE::QMPClient->new();
4759
4760             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4761         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4762             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4763                 if scalar(%{$cmd->{arguments}});
4764             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4765         } else {
4766             die "unable to open monitor socket\n";
4767         }
4768     };
4769     if (my $err = $@) {
4770         syslog("err", "VM $vmid qmp command failed - $err");
4771         die $err;
4772     }
4773
4774     return $res;
4775 }
4776
4777 sub vm_human_monitor_command {
4778     my ($vmid, $cmdline) = @_;
4779
4780     my $res;
4781
4782     my $cmd = {
4783         execute => 'human-monitor-command',
4784         arguments => { 'command-line' => $cmdline},
4785     };
4786
4787     return vm_qmp_command($vmid, $cmd);
4788 }
4789
4790 sub vm_commandline {
4791     my ($storecfg, $vmid) = @_;
4792
4793     my $conf = PVE::QemuConfig->load_config($vmid);
4794
4795     my $defaults = load_defaults();
4796
4797     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4798
4799     return PVE::Tools::cmd2string($cmd);
4800 }
4801
4802 sub vm_reset {
4803     my ($vmid, $skiplock) = @_;
4804
4805     PVE::QemuConfig->lock_config($vmid, sub {
4806
4807         my $conf = PVE::QemuConfig->load_config($vmid);
4808
4809         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4810
4811         vm_mon_cmd($vmid, "system_reset");
4812     });
4813 }
4814
4815 sub get_vm_volumes {
4816     my ($conf) = @_;
4817
4818     my $vollist = [];
4819     foreach_volid($conf, sub {
4820         my ($volid, $is_cdrom) = @_;
4821
4822         return if $volid =~ m|^/|;
4823
4824         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4825         return if !$sid;
4826
4827         push @$vollist, $volid;
4828     });
4829
4830     return $vollist;
4831 }
4832
4833 sub vm_stop_cleanup {
4834     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4835
4836     eval {
4837
4838         if (!$keepActive) {
4839             my $vollist = get_vm_volumes($conf);
4840             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4841         }
4842
4843         foreach my $ext (qw(mon qmp pid vnc qga)) {
4844             unlink "/var/run/qemu-server/${vmid}.$ext";
4845         }
4846
4847         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4848     };
4849     warn $@ if $@; # avoid errors - just warn
4850 }
4851
4852 # Note: use $nockeck to skip tests if VM configuration file exists.
4853 # We need that when migration VMs to other nodes (files already moved)
4854 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4855 sub vm_stop {
4856     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4857
4858     $force = 1 if !defined($force) && !$shutdown;
4859
4860     if ($migratedfrom){
4861         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4862         kill 15, $pid if $pid;
4863         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4864         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4865         return;
4866     }
4867
4868     PVE::QemuConfig->lock_config($vmid, sub {
4869
4870         my $pid = check_running($vmid, $nocheck);
4871         return if !$pid;
4872
4873         my $conf;
4874         if (!$nocheck) {
4875             $conf = PVE::QemuConfig->load_config($vmid);
4876             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4877             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4878                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4879                 $timeout = $opts->{down} if $opts->{down};
4880             }
4881         }
4882
4883         $timeout = 60 if !defined($timeout);
4884
4885         eval {
4886             if ($shutdown) {
4887                 if (defined($conf) && $conf->{agent}) {
4888                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4889                 } else {
4890                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4891                 }
4892             } else {
4893                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4894             }
4895         };
4896         my $err = $@;
4897
4898         if (!$err) {
4899             my $count = 0;
4900             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4901                 $count++;
4902                 sleep 1;
4903             }
4904
4905             if ($count >= $timeout) {
4906                 if ($force) {
4907                     warn "VM still running - terminating now with SIGTERM\n";
4908                     kill 15, $pid;
4909                 } else {
4910                     die "VM quit/powerdown failed - got timeout\n";
4911                 }
4912             } else {
4913                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4914                 return;
4915             }
4916         } else {
4917             if ($force) {
4918                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4919                 kill 15, $pid;
4920             } else {
4921                 die "VM quit/powerdown failed\n";
4922             }
4923         }
4924
4925         # wait again
4926         $timeout = 10;
4927
4928         my $count = 0;
4929         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4930             $count++;
4931             sleep 1;
4932         }
4933
4934         if ($count >= $timeout) {
4935             warn "VM still running - terminating now with SIGKILL\n";
4936             kill 9, $pid;
4937             sleep 1;
4938         }
4939
4940         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4941    });
4942 }
4943
4944 sub vm_suspend {
4945     my ($vmid, $skiplock) = @_;
4946
4947     PVE::QemuConfig->lock_config($vmid, sub {
4948
4949         my $conf = PVE::QemuConfig->load_config($vmid);
4950
4951         PVE::QemuConfig->check_lock($conf)
4952             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4953
4954         vm_mon_cmd($vmid, "stop");
4955     });
4956 }
4957
4958 sub vm_resume {
4959     my ($vmid, $skiplock, $nocheck) = @_;
4960
4961     PVE::QemuConfig->lock_config($vmid, sub {
4962
4963         if (!$nocheck) {
4964
4965             my $conf = PVE::QemuConfig->load_config($vmid);
4966
4967             PVE::QemuConfig->check_lock($conf)
4968                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4969
4970             vm_mon_cmd($vmid, "cont");
4971
4972         } else {
4973             vm_mon_cmd_nocheck($vmid, "cont");
4974         }
4975     });
4976 }
4977
4978 sub vm_sendkey {
4979     my ($vmid, $skiplock, $key) = @_;
4980
4981     PVE::QemuConfig->lock_config($vmid, sub {
4982
4983         my $conf = PVE::QemuConfig->load_config($vmid);
4984
4985         # there is no qmp command, so we use the human monitor command
4986         vm_human_monitor_command($vmid, "sendkey $key");
4987     });
4988 }
4989
4990 sub vm_destroy {
4991     my ($storecfg, $vmid, $skiplock) = @_;
4992
4993     PVE::QemuConfig->lock_config($vmid, sub {
4994
4995         my $conf = PVE::QemuConfig->load_config($vmid);
4996
4997         if (!check_running($vmid)) {
4998             destroy_vm($storecfg, $vmid, undef, $skiplock);
4999         } else {
5000             die "VM $vmid is running - destroy failed\n";
5001         }
5002     });
5003 }
5004
5005 # pci helpers
5006
5007 sub file_write {
5008     my ($filename, $buf) = @_;
5009
5010     my $fh = IO::File->new($filename, "w");
5011     return undef if !$fh;
5012
5013     my $res = print $fh $buf;
5014
5015     $fh->close();
5016
5017     return $res;
5018 }
5019
5020 sub pci_device_info {
5021     my ($name) = @_;
5022
5023     my $res;
5024
5025     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5026     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5027
5028     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5029     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5030
5031     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5032     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5033
5034     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5035     return undef if !defined($product) || $product !~ s/^0x//;
5036
5037     $res = {
5038         name => $name,
5039         vendor => $vendor,
5040         product => $product,
5041         domain => $domain,
5042         bus => $bus,
5043         slot => $slot,
5044         func => $func,
5045         irq => $irq,
5046         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5047     };
5048
5049     return $res;
5050 }
5051
5052 sub pci_dev_reset {
5053     my ($dev) = @_;
5054
5055     my $name = $dev->{name};
5056
5057     my $fn = "$pcisysfs/devices/$name/reset";
5058
5059     return file_write($fn, "1");
5060 }
5061
5062 sub pci_dev_bind_to_vfio {
5063     my ($dev) = @_;
5064
5065     my $name = $dev->{name};
5066
5067     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5068
5069     if (!-d $vfio_basedir) {
5070         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5071     }
5072     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5073
5074     my $testdir = "$vfio_basedir/$name";
5075     return 1 if -d $testdir;
5076
5077     my $data = "$dev->{vendor} $dev->{product}";
5078     return undef if !file_write("$vfio_basedir/new_id", $data);
5079
5080     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5081     if (!file_write($fn, $name)) {
5082         return undef if -f $fn;
5083     }
5084
5085     $fn = "$vfio_basedir/bind";
5086     if (! -d $testdir) {
5087         return undef if !file_write($fn, $name);
5088     }
5089
5090     return -d $testdir;
5091 }
5092
5093 sub pci_dev_group_bind_to_vfio {
5094     my ($pciid) = @_;
5095
5096     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5097
5098     if (!-d $vfio_basedir) {
5099         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5100     }
5101     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5102
5103     # get IOMMU group devices
5104     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5105       my @devs = grep /^0000:/, readdir($D);
5106     closedir($D);
5107
5108     foreach my $pciid (@devs) {
5109         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5110
5111         # pci bridges, switches or root ports are not supported
5112         # they have a pci_bus subdirectory so skip them
5113         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5114
5115         my $info = pci_device_info($1);
5116         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5117     }
5118
5119     return 1;
5120 }
5121
5122 # vzdump restore implementaion
5123
5124 sub tar_archive_read_firstfile {
5125     my $archive = shift;
5126
5127     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5128
5129     # try to detect archive type first
5130     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5131         die "unable to open file '$archive'\n";
5132     my $firstfile = <$fh>;
5133     kill 15, $pid;
5134     close $fh;
5135
5136     die "ERROR: archive contaions no data\n" if !$firstfile;
5137     chomp $firstfile;
5138
5139     return $firstfile;
5140 }
5141
5142 sub tar_restore_cleanup {
5143     my ($storecfg, $statfile) = @_;
5144
5145     print STDERR "starting cleanup\n";
5146
5147     if (my $fd = IO::File->new($statfile, "r")) {
5148         while (defined(my $line = <$fd>)) {
5149             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5150                 my $volid = $2;
5151                 eval {
5152                     if ($volid =~ m|^/|) {
5153                         unlink $volid || die 'unlink failed\n';
5154                     } else {
5155                         PVE::Storage::vdisk_free($storecfg, $volid);
5156                     }
5157                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5158                 };
5159                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5160             } else {
5161                 print STDERR "unable to parse line in statfile - $line";
5162             }
5163         }
5164         $fd->close();
5165     }
5166 }
5167
5168 sub restore_archive {
5169     my ($archive, $vmid, $user, $opts) = @_;
5170
5171     my $format = $opts->{format};
5172     my $comp;
5173
5174     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5175         $format = 'tar' if !$format;
5176         $comp = 'gzip';
5177     } elsif ($archive =~ m/\.tar$/) {
5178         $format = 'tar' if !$format;
5179     } elsif ($archive =~ m/.tar.lzo$/) {
5180         $format = 'tar' if !$format;
5181         $comp = 'lzop';
5182     } elsif ($archive =~ m/\.vma$/) {
5183         $format = 'vma' if !$format;
5184     } elsif ($archive =~ m/\.vma\.gz$/) {
5185         $format = 'vma' if !$format;
5186         $comp = 'gzip';
5187     } elsif ($archive =~ m/\.vma\.lzo$/) {
5188         $format = 'vma' if !$format;
5189         $comp = 'lzop';
5190     } else {
5191         $format = 'vma' if !$format; # default
5192     }
5193
5194     # try to detect archive format
5195     if ($format eq 'tar') {
5196         return restore_tar_archive($archive, $vmid, $user, $opts);
5197     } else {
5198         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5199     }
5200 }
5201
5202 sub restore_update_config_line {
5203     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5204
5205     return if $line =~ m/^\#qmdump\#/;
5206     return if $line =~ m/^\#vzdump\#/;
5207     return if $line =~ m/^lock:/;
5208     return if $line =~ m/^unused\d+:/;
5209     return if $line =~ m/^parent:/;
5210     return if $line =~ m/^template:/; # restored VM is never a template
5211
5212     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5213     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5214         # try to convert old 1.X settings
5215         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5216         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5217             my ($model, $macaddr) = split(/\=/, $devconfig);
5218             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5219             my $net = {
5220                 model => $model,
5221                 bridge => "vmbr$ind",
5222                 macaddr => $macaddr,
5223             };
5224             my $netstr = print_net($net);
5225
5226             print $outfd "net$cookie->{netcount}: $netstr\n";
5227             $cookie->{netcount}++;
5228         }
5229     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5230         my ($id, $netstr) = ($1, $2);
5231         my $net = parse_net($netstr);
5232         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5233         $netstr = print_net($net);
5234         print $outfd "$id: $netstr\n";
5235     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5236         my $virtdev = $1;
5237         my $value = $3;
5238         my $di = parse_drive($virtdev, $value);
5239         if (defined($di->{backup}) && !$di->{backup}) {
5240             print $outfd "#$line";
5241         } elsif ($map->{$virtdev}) {
5242             delete $di->{format}; # format can change on restore
5243             $di->{file} = $map->{$virtdev};
5244             $value = print_drive($vmid, $di);
5245             print $outfd "$virtdev: $value\n";
5246         } else {
5247             print $outfd $line;
5248         }
5249     } else {
5250         print $outfd $line;
5251     }
5252 }
5253
5254 sub scan_volids {
5255     my ($cfg, $vmid) = @_;
5256
5257     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5258
5259     my $volid_hash = {};
5260     foreach my $storeid (keys %$info) {
5261         foreach my $item (@{$info->{$storeid}}) {
5262             next if !($item->{volid} && $item->{size});
5263             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5264             $volid_hash->{$item->{volid}} = $item;
5265         }
5266     }
5267
5268     return $volid_hash;
5269 }
5270
5271 sub is_volume_in_use {
5272     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5273
5274     my $path = PVE::Storage::path($storecfg, $volid);
5275
5276     my $scan_config = sub {
5277         my ($cref, $snapname) = @_;
5278
5279         foreach my $key (keys %$cref) {
5280             my $value = $cref->{$key};
5281             if (is_valid_drivename($key)) {
5282                 next if $skip_drive && $key eq $skip_drive;
5283                 my $drive = parse_drive($key, $value);
5284                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5285                 return 1 if $volid eq $drive->{file};
5286                 if ($drive->{file} =~ m!^/!) {
5287                     return 1 if $drive->{file} eq $path;
5288                 } else {
5289                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5290                     next if !$storeid;
5291                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5292                     next if !$scfg;
5293                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5294                 }
5295             }
5296         }
5297
5298         return 0;
5299     };
5300
5301     return 1 if &$scan_config($conf);
5302
5303     undef $skip_drive;
5304
5305     foreach my $snapname (keys %{$conf->{snapshots}}) {
5306         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5307     }
5308
5309     return 0;
5310 }
5311
5312 sub update_disksize {
5313     my ($vmid, $conf, $volid_hash) = @_;
5314
5315     my $changes;
5316
5317     my $used = {};
5318
5319     # Note: it is allowed to define multiple storages with same path (alias), so
5320     # we need to check both 'volid' and real 'path' (two different volid can point
5321     # to the same path).
5322
5323     my $usedpath = {};
5324
5325     # update size info
5326     foreach my $opt (keys %$conf) {
5327         if (is_valid_drivename($opt)) {
5328             my $drive = parse_drive($opt, $conf->{$opt});
5329             my $volid = $drive->{file};
5330             next if !$volid;
5331
5332             $used->{$volid} = 1;
5333             if ($volid_hash->{$volid} &&
5334                 (my $path = $volid_hash->{$volid}->{path})) {
5335                 $usedpath->{$path} = 1;
5336             }
5337
5338             next if drive_is_cdrom($drive);
5339             next if !$volid_hash->{$volid};
5340
5341             $drive->{size} = $volid_hash->{$volid}->{size};
5342             my $new = print_drive($vmid, $drive);
5343             if ($new ne $conf->{$opt}) {
5344                 $changes = 1;
5345                 $conf->{$opt} = $new;
5346             }
5347         }
5348     }
5349
5350     # remove 'unusedX' entry if volume is used
5351     foreach my $opt (keys %$conf) {
5352         next if $opt !~ m/^unused\d+$/;
5353         my $volid = $conf->{$opt};
5354         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5355         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5356             $changes = 1;
5357             delete $conf->{$opt};
5358         }
5359     }
5360
5361     foreach my $volid (sort keys %$volid_hash) {
5362         next if $volid =~ m/vm-$vmid-state-/;
5363         next if $used->{$volid};
5364         my $path = $volid_hash->{$volid}->{path};
5365         next if !$path; # just to be sure
5366         next if $usedpath->{$path};
5367         $changes = 1;
5368         PVE::QemuConfig->add_unused_volume($conf, $volid);
5369         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5370     }
5371
5372     return $changes;
5373 }
5374
5375 sub rescan {
5376     my ($vmid, $nolock) = @_;
5377
5378     my $cfg = PVE::Storage::config();
5379
5380     my $volid_hash = scan_volids($cfg, $vmid);
5381
5382     my $updatefn =  sub {
5383         my ($vmid) = @_;
5384
5385         my $conf = PVE::QemuConfig->load_config($vmid);
5386
5387         PVE::QemuConfig->check_lock($conf);
5388
5389         my $vm_volids = {};
5390         foreach my $volid (keys %$volid_hash) {
5391             my $info = $volid_hash->{$volid};
5392             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5393         }
5394
5395         my $changes = update_disksize($vmid, $conf, $vm_volids);
5396
5397         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5398     };
5399
5400     if (defined($vmid)) {
5401         if ($nolock) {
5402             &$updatefn($vmid);
5403         } else {
5404             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5405         }
5406     } else {
5407         my $vmlist = config_list();
5408         foreach my $vmid (keys %$vmlist) {
5409             if ($nolock) {
5410                 &$updatefn($vmid);
5411             } else {
5412                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5413             }
5414         }
5415     }
5416 }
5417
5418 sub restore_vma_archive {
5419     my ($archive, $vmid, $user, $opts, $comp) = @_;
5420
5421     my $input = $archive eq '-' ? "<&STDIN" : undef;
5422     my $readfrom = $archive;
5423
5424     my $uncomp = '';
5425     if ($comp) {
5426         $readfrom = '-';
5427         my $qarchive = PVE::Tools::shellquote($archive);
5428         if ($comp eq 'gzip') {
5429             $uncomp = "zcat $qarchive|";
5430         } elsif ($comp eq 'lzop') {
5431             $uncomp = "lzop -d -c $qarchive|";
5432         } else {
5433             die "unknown compression method '$comp'\n";
5434         }
5435
5436     }
5437
5438     my $tmpdir = "/var/tmp/vzdumptmp$$";
5439     rmtree $tmpdir;
5440
5441     # disable interrupts (always do cleanups)
5442     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5443         warn "got interrupt - ignored\n";
5444     };
5445
5446     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5447     POSIX::mkfifo($mapfifo, 0600);
5448     my $fifofh;
5449
5450     my $openfifo = sub {
5451         open($fifofh, '>', $mapfifo) || die $!;
5452     };
5453
5454     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5455
5456     my $oldtimeout;
5457     my $timeout = 5;
5458
5459     my $devinfo = {};
5460
5461     my $rpcenv = PVE::RPCEnvironment::get();
5462
5463     my $conffile = PVE::QemuConfig->config_file($vmid);
5464     my $tmpfn = "$conffile.$$.tmp";
5465
5466     # Note: $oldconf is undef if VM does not exists
5467     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5468     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5469
5470     my $print_devmap = sub {
5471         my $virtdev_hash = {};
5472
5473         my $cfgfn = "$tmpdir/qemu-server.conf";
5474
5475         # we can read the config - that is already extracted
5476         my $fh = IO::File->new($cfgfn, "r") ||
5477             "unable to read qemu-server.conf - $!\n";
5478
5479         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5480         if (-f $fwcfgfn) {
5481             my $pve_firewall_dir = '/etc/pve/firewall';
5482             mkdir $pve_firewall_dir; # make sure the dir exists
5483             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5484         }
5485
5486         while (defined(my $line = <$fh>)) {
5487             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5488                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5489                 die "archive does not contain data for drive '$virtdev'\n"
5490                     if !$devinfo->{$devname};
5491                 if (defined($opts->{storage})) {
5492                     $storeid = $opts->{storage} || 'local';
5493                 } elsif (!$storeid) {
5494                     $storeid = 'local';
5495                 }
5496                 $format = 'raw' if !$format;
5497                 $devinfo->{$devname}->{devname} = $devname;
5498                 $devinfo->{$devname}->{virtdev} = $virtdev;
5499                 $devinfo->{$devname}->{format} = $format;
5500                 $devinfo->{$devname}->{storeid} = $storeid;
5501
5502                 # check permission on storage
5503                 my $pool = $opts->{pool}; # todo: do we need that?
5504                 if ($user ne 'root@pam') {
5505                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5506                 }
5507
5508                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5509             }
5510         }
5511
5512         foreach my $devname (keys %$devinfo) {
5513             die "found no device mapping information for device '$devname'\n"
5514                 if !$devinfo->{$devname}->{virtdev};
5515         }
5516
5517         my $cfg = PVE::Storage::config();
5518
5519         # create empty/temp config
5520         if ($oldconf) {
5521             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5522             foreach_drive($oldconf, sub {
5523                 my ($ds, $drive) = @_;
5524
5525                 return if drive_is_cdrom($drive);
5526
5527                 my $volid = $drive->{file};
5528
5529                 return if !$volid || $volid =~ m|^/|;
5530
5531                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5532                 return if !$path || !$owner || ($owner != $vmid);
5533
5534                 # Note: only delete disk we want to restore
5535                 # other volumes will become unused
5536                 if ($virtdev_hash->{$ds}) {
5537                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5538                     if (my $err = $@) {
5539                         warn $err;
5540                     }
5541                 }
5542             });
5543
5544             # delete vmstate files
5545             # since after the restore we have no snapshots anymore
5546             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5547                 my $snap = $oldconf->{snapshots}->{$snapname};
5548                 if ($snap->{vmstate}) {
5549                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5550                     if (my $err = $@) {
5551                         warn $err;
5552                     }
5553                 }
5554             }
5555         }
5556
5557         my $map = {};
5558         foreach my $virtdev (sort keys %$virtdev_hash) {
5559             my $d = $virtdev_hash->{$virtdev};
5560             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5561             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5562
5563             # test if requested format is supported
5564             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5565             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5566             $d->{format} = $defFormat if !$supported;
5567
5568             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5569                                                   $d->{format}, undef, $alloc_size);
5570             print STDERR "new volume ID is '$volid'\n";
5571             $d->{volid} = $volid;
5572             my $path = PVE::Storage::path($cfg, $volid);
5573
5574             PVE::Storage::activate_volumes($cfg,[$volid]);
5575
5576             my $write_zeros = 1;
5577             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5578                 $write_zeros = 0;
5579             }
5580
5581             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5582
5583             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5584             $map->{$virtdev} = $volid;
5585         }
5586
5587         $fh->seek(0, 0) || die "seek failed - $!\n";
5588
5589         my $outfd = new IO::File ($tmpfn, "w") ||
5590             die "unable to write config for VM $vmid\n";
5591
5592         my $cookie = { netcount => 0 };
5593         while (defined(my $line = <$fh>)) {
5594             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5595         }
5596
5597         $fh->close();
5598         $outfd->close();
5599     };
5600
5601     eval {
5602         # enable interrupts
5603         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5604             die "interrupted by signal\n";
5605         };
5606         local $SIG{ALRM} = sub { die "got timeout\n"; };
5607
5608         $oldtimeout = alarm($timeout);
5609
5610         my $parser = sub {
5611             my $line = shift;
5612
5613             print "$line\n";
5614
5615             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5616                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5617                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5618             } elsif ($line =~ m/^CTIME: /) {
5619                 # we correctly received the vma config, so we can disable
5620                 # the timeout now for disk allocation (set to 10 minutes, so
5621                 # that we always timeout if something goes wrong)
5622                 alarm(600);
5623                 &$print_devmap();
5624                 print $fifofh "done\n";
5625                 my $tmp = $oldtimeout || 0;
5626                 $oldtimeout = undef;
5627                 alarm($tmp);
5628                 close($fifofh);
5629             }
5630         };
5631
5632         print "restore vma archive: $cmd\n";
5633         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5634     };
5635     my $err = $@;
5636
5637     alarm($oldtimeout) if $oldtimeout;
5638
5639     my $vollist = [];
5640     foreach my $devname (keys %$devinfo) {
5641         my $volid = $devinfo->{$devname}->{volid};
5642         push @$vollist, $volid if $volid;
5643     }
5644
5645     my $cfg = PVE::Storage::config();
5646     PVE::Storage::deactivate_volumes($cfg, $vollist);
5647
5648     unlink $mapfifo;
5649
5650     if ($err) {
5651         rmtree $tmpdir;
5652         unlink $tmpfn;
5653
5654         foreach my $devname (keys %$devinfo) {
5655             my $volid = $devinfo->{$devname}->{volid};
5656             next if !$volid;
5657             eval {
5658                 if ($volid =~ m|^/|) {
5659                     unlink $volid || die 'unlink failed\n';
5660                 } else {
5661                     PVE::Storage::vdisk_free($cfg, $volid);
5662                 }
5663                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5664             };
5665             print STDERR "unable to cleanup '$volid' - $@" if $@;
5666         }
5667         die $err;
5668     }
5669
5670     rmtree $tmpdir;
5671
5672     rename($tmpfn, $conffile) ||
5673         die "unable to commit configuration file '$conffile'\n";
5674
5675     PVE::Cluster::cfs_update(); # make sure we read new file
5676
5677     eval { rescan($vmid, 1); };
5678     warn $@ if $@;
5679 }
5680
5681 sub restore_tar_archive {
5682     my ($archive, $vmid, $user, $opts) = @_;
5683
5684     if ($archive ne '-') {
5685         my $firstfile = tar_archive_read_firstfile($archive);
5686         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5687             if $firstfile ne 'qemu-server.conf';
5688     }
5689
5690     my $storecfg = PVE::Storage::config();
5691
5692     # destroy existing data - keep empty config
5693     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5694     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5695
5696     my $tocmd = "/usr/lib/qemu-server/qmextract";
5697
5698     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5699     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5700     $tocmd .= ' --prealloc' if $opts->{prealloc};
5701     $tocmd .= ' --info' if $opts->{info};
5702
5703     # tar option "xf" does not autodetect compression when read from STDIN,
5704     # so we pipe to zcat
5705     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5706         PVE::Tools::shellquote("--to-command=$tocmd");
5707
5708     my $tmpdir = "/var/tmp/vzdumptmp$$";
5709     mkpath $tmpdir;
5710
5711     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5712     local $ENV{VZDUMP_VMID} = $vmid;
5713     local $ENV{VZDUMP_USER} = $user;
5714
5715     my $conffile = PVE::QemuConfig->config_file($vmid);
5716     my $tmpfn = "$conffile.$$.tmp";
5717
5718     # disable interrupts (always do cleanups)
5719     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5720         print STDERR "got interrupt - ignored\n";
5721     };
5722
5723     eval {
5724         # enable interrupts
5725         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5726             die "interrupted by signal\n";
5727         };
5728
5729         if ($archive eq '-') {
5730             print "extracting archive from STDIN\n";
5731             run_command($cmd, input => "<&STDIN");
5732         } else {
5733             print "extracting archive '$archive'\n";
5734             run_command($cmd);
5735         }
5736
5737         return if $opts->{info};
5738
5739         # read new mapping
5740         my $map = {};
5741         my $statfile = "$tmpdir/qmrestore.stat";
5742         if (my $fd = IO::File->new($statfile, "r")) {
5743             while (defined (my $line = <$fd>)) {
5744                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5745                     $map->{$1} = $2 if $1;
5746                 } else {
5747                     print STDERR "unable to parse line in statfile - $line\n";
5748                 }
5749             }
5750             $fd->close();
5751         }
5752
5753         my $confsrc = "$tmpdir/qemu-server.conf";
5754
5755         my $srcfd = new IO::File($confsrc, "r") ||
5756             die "unable to open file '$confsrc'\n";
5757
5758         my $outfd = new IO::File ($tmpfn, "w") ||
5759             die "unable to write config for VM $vmid\n";
5760
5761         my $cookie = { netcount => 0 };
5762         while (defined (my $line = <$srcfd>)) {
5763             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5764         }
5765
5766         $srcfd->close();
5767         $outfd->close();
5768     };
5769     my $err = $@;
5770
5771     if ($err) {
5772
5773         unlink $tmpfn;
5774
5775         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5776
5777         die $err;
5778     }
5779
5780     rmtree $tmpdir;
5781
5782     rename $tmpfn, $conffile ||
5783         die "unable to commit configuration file '$conffile'\n";
5784
5785     PVE::Cluster::cfs_update(); # make sure we read new file
5786
5787     eval { rescan($vmid, 1); };
5788     warn $@ if $@;
5789 };
5790
5791 sub foreach_writable_storage {
5792     my ($conf, $func) = @_;
5793
5794     my $sidhash = {};
5795
5796     foreach my $ds (keys %$conf) {
5797         next if !is_valid_drivename($ds);
5798
5799         my $drive = parse_drive($ds, $conf->{$ds});
5800         next if !$drive;
5801         next if drive_is_cdrom($drive);
5802
5803         my $volid = $drive->{file};
5804
5805         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5806         $sidhash->{$sid} = $sid if $sid;
5807     }
5808
5809     foreach my $sid (sort keys %$sidhash) {
5810         &$func($sid);
5811     }
5812 }
5813
5814 sub do_snapshots_with_qemu {
5815     my ($storecfg, $volid) = @_;
5816
5817     my $storage_name = PVE::Storage::parse_volume_id($volid);
5818
5819     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5820         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5821         return 1;
5822     }
5823
5824     if ($volid =~ m/\.(qcow2|qed)$/){
5825         return 1;
5826     }
5827
5828     return undef;
5829 }
5830
5831 sub qga_check_running {
5832     my ($vmid) = @_;
5833
5834     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5835     if ($@) {
5836         warn "Qemu Guest Agent are not running - $@";
5837         return 0;
5838     }
5839     return 1;
5840 }
5841
5842 sub template_create {
5843     my ($vmid, $conf, $disk) = @_;
5844
5845     my $storecfg = PVE::Storage::config();
5846
5847     foreach_drive($conf, sub {
5848         my ($ds, $drive) = @_;
5849
5850         return if drive_is_cdrom($drive);
5851         return if $disk && $ds ne $disk;
5852
5853         my $volid = $drive->{file};
5854         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5855
5856         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5857         $drive->{file} = $voliddst;
5858         $conf->{$ds} = print_drive($vmid, $drive);
5859         PVE::QemuConfig->write_config($vmid, $conf);
5860     });
5861 }
5862
5863 sub qemu_img_convert {
5864     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5865
5866     my $storecfg = PVE::Storage::config();
5867     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5868     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5869
5870     if ($src_storeid && $dst_storeid) {
5871
5872         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5873
5874         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5875         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5876
5877         my $src_format = qemu_img_format($src_scfg, $src_volname);
5878         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5879
5880         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5881         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5882
5883         my $cmd = [];
5884         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5885         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5886         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5887         if ($is_zero_initialized) {
5888             push @$cmd, "zeroinit:$dst_path";
5889         } else {
5890             push @$cmd, $dst_path;
5891         }
5892
5893         my $parser = sub {
5894             my $line = shift;
5895             if($line =~ m/\((\S+)\/100\%\)/){
5896                 my $percent = $1;
5897                 my $transferred = int($size * $percent / 100);
5898                 my $remaining = $size - $transferred;
5899
5900                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5901             }
5902
5903         };
5904
5905         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5906         my $err = $@;
5907         die "copy failed: $err" if $err;
5908     }
5909 }
5910
5911 sub qemu_img_format {
5912     my ($scfg, $volname) = @_;
5913
5914     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5915         return $1;
5916     } else {
5917         return "raw";
5918     }
5919 }
5920
5921 sub qemu_drive_mirror {
5922     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
5923
5924     $jobs = {} if !$jobs;
5925
5926     my $qemu_target;
5927     my $format;
5928     $jobs->{"drive-$drive"} = {};
5929
5930     if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
5931         my $server = $1;
5932         my $port = $2;
5933         my $exportname = $3;
5934
5935         $format = "nbd";
5936         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
5937         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
5938         my $cmd = ['socat', '-T30', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=5"];
5939
5940         my $pid = fork();
5941         if (!defined($pid)) {
5942             die "forking socat tunnel failed\n";
5943         } elsif ($pid == 0) {
5944             exec(@$cmd);
5945             warn "exec failed: $!\n";
5946             POSIX::_exit(-1);
5947         }
5948         $jobs->{"drive-$drive"}->{pid} = $pid;
5949
5950         my $timeout = 0;
5951         while (!-S $unixsocket) {
5952             die "nbd connection helper timed out\n"
5953                 if $timeout++ > 5;
5954             sleep 1;
5955         }
5956     } else {
5957         my $storecfg = PVE::Storage::config();
5958         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5959
5960         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5961
5962         $format = qemu_img_format($dst_scfg, $dst_volname);
5963
5964         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5965
5966         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5967     }
5968
5969     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5970     $opts->{format} = $format if $format;
5971
5972     print "drive mirror is starting for drive-$drive\n";
5973
5974     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
5975
5976     if (my $err = $@) {
5977         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
5978         die "mirroring error: $err";
5979     }
5980
5981     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
5982 }
5983
5984 sub qemu_drive_mirror_monitor {
5985     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
5986
5987     eval {
5988         my $err_complete = 0;
5989
5990         while (1) {
5991             die "storage migration timed out\n" if $err_complete > 300;
5992
5993             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5994
5995             my $running_mirror_jobs = {};
5996             foreach my $stat (@$stats) {
5997                 next if $stat->{type} ne 'mirror';
5998                 $running_mirror_jobs->{$stat->{device}} = $stat;
5999             }
6000
6001             my $readycounter = 0;
6002
6003             foreach my $job (keys %$jobs) {
6004
6005                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6006                     print "$job : finished\n";
6007                     delete $jobs->{$job};
6008                     next;
6009                 }
6010
6011                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6012
6013                 my $busy = $running_mirror_jobs->{$job}->{busy};
6014                 my $ready = $running_mirror_jobs->{$job}->{ready};
6015                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6016                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6017                     my $remaining = $total - $transferred;
6018                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6019
6020                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6021                 }
6022
6023                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6024             }
6025
6026             last if scalar(keys %$jobs) == 0;
6027
6028             if ($readycounter == scalar(keys %$jobs)) {
6029                 print "all mirroring jobs are ready \n";
6030                 last if $skipcomplete; #do the complete later
6031
6032                 if ($vmiddst && $vmiddst != $vmid) {
6033                     if ($qga) {
6034                         print "freeze filesystem\n";
6035                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6036                     } else {
6037                         print "suspend vm\n";
6038                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6039                     }
6040
6041                     # if we clone a disk for a new target vm, we don't switch the disk
6042                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6043
6044                     if ($qga) {
6045                         print "unfreeze filesystem\n";
6046                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6047                     } else {
6048                         print "resume vm\n";
6049                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6050                     }
6051
6052                     last;
6053                 } else {
6054
6055                     foreach my $job (keys %$jobs) {
6056                         # try to switch the disk if source and destination are on the same guest
6057                         print "$job: Completing block job...\n";
6058
6059                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6060                         if ($@ =~ m/cannot be completed/) {
6061                             print "$job: Block job cannot be completed, try again.\n";
6062                             $err_complete++;
6063                         }else {
6064                             print "$job: Completed successfully.\n";
6065                             $jobs->{$job}->{complete} = 1;
6066                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6067                         }
6068                     }
6069                 }
6070             }
6071             sleep 1;
6072         }
6073     };
6074     my $err = $@;
6075
6076     if ($err) {
6077         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6078         die "mirroring error: $err";
6079     }
6080
6081 }
6082
6083 sub qemu_blockjobs_cancel {
6084     my ($vmid, $jobs) = @_;
6085
6086     foreach my $job (keys %$jobs) {
6087         print "$job: Cancelling block job\n";
6088         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6089         $jobs->{$job}->{cancel} = 1;
6090     }
6091
6092     while (1) {
6093         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6094
6095         my $running_jobs = {};
6096         foreach my $stat (@$stats) {
6097             $running_jobs->{$stat->{device}} = $stat;
6098         }
6099
6100         foreach my $job (keys %$jobs) {
6101
6102             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6103                 print "$job: Done.\n";
6104                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6105                 delete $jobs->{$job};
6106             }
6107         }
6108
6109         last if scalar(keys %$jobs) == 0;
6110
6111         sleep 1;
6112     }
6113 }
6114
6115 sub qemu_blockjobs_finish_tunnel {
6116    my ($vmid, $job, $cpid) = @_;
6117
6118    return if !$cpid;
6119
6120    for (my $i = 1; $i < 20; $i++) {
6121         my $waitpid = waitpid($cpid, WNOHANG);
6122         last if (defined($waitpid) && ($waitpid == $cpid));
6123
6124         if ($i == 10) {
6125             kill(15, $cpid);
6126         } elsif ($i >= 15) {
6127             kill(9, $cpid);
6128         }
6129         sleep (1);
6130     }
6131     unlink "/run/qemu-server/$vmid.mirror-$job";
6132 }
6133
6134 sub clone_disk {
6135     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6136         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6137
6138     my $newvolid;
6139
6140     if (!$full) {
6141         print "create linked clone of drive $drivename ($drive->{file})\n";
6142         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6143         push @$newvollist, $newvolid;
6144     } else {
6145
6146         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6147         $storeid = $storage if $storage;
6148
6149         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6150         if (!$format) {
6151             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6152             $format = qemu_img_format($scfg, $volname);
6153         }
6154
6155         # test if requested format is supported - else use default
6156         my $supported = grep { $_ eq $format } @$validFormats;
6157         $format = $defFormat if !$supported;
6158
6159         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6160
6161         print "create full clone of drive $drivename ($drive->{file})\n";
6162         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6163         push @$newvollist, $newvolid;
6164
6165         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6166
6167         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6168         if (!$running || $snapname) {
6169             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6170         } else {
6171
6172             my $kvmver = get_running_qemu_version ($vmid);
6173             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6174                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6175                     if $drive->{iothread};
6176             }
6177
6178             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6179         }
6180     }
6181
6182     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6183
6184     my $disk = $drive;
6185     $disk->{format} = undef;
6186     $disk->{file} = $newvolid;
6187     $disk->{size} = $size;
6188
6189     return $disk;
6190 }
6191
6192 # this only works if VM is running
6193 sub get_current_qemu_machine {
6194     my ($vmid) = @_;
6195
6196     my $cmd = { execute => 'query-machines', arguments => {} };
6197     my $res = vm_qmp_command($vmid, $cmd);
6198
6199     my ($current, $default);
6200     foreach my $e (@$res) {
6201         $default = $e->{name} if $e->{'is-default'};
6202         $current = $e->{name} if $e->{'is-current'};
6203     }
6204
6205     # fallback to the default machine if current is not supported by qemu
6206     return $current || $default || 'pc';
6207 }
6208
6209 sub get_running_qemu_version {
6210     my ($vmid) = @_;
6211     my $cmd = { execute => 'query-version', arguments => {} };
6212     my $res = vm_qmp_command($vmid, $cmd);
6213     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6214 }
6215
6216 sub qemu_machine_feature_enabled {
6217     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6218
6219     my $current_major;
6220     my $current_minor;
6221
6222     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6223
6224         $current_major = $3;
6225         $current_minor = $4;
6226
6227     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6228
6229         $current_major = $1;
6230         $current_minor = $2;
6231     }
6232
6233     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6234
6235
6236 }
6237
6238 sub qemu_machine_pxe {
6239     my ($vmid, $conf, $machine) = @_;
6240
6241     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6242
6243     foreach my $opt (keys %$conf) {
6244         next if $opt !~ m/^net(\d+)$/;
6245         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6246         next if !$net;
6247         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6248         return $machine.".pxe" if $romfile =~ m/pxe/;
6249         last;
6250     }
6251
6252     return $machine;
6253 }
6254
6255 sub qemu_use_old_bios_files {
6256     my ($machine_type) = @_;
6257
6258     return if !$machine_type;
6259
6260     my $use_old_bios_files = undef;
6261
6262     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6263         $machine_type = $1;
6264         $use_old_bios_files = 1;
6265     } else {
6266         my $kvmver = kvm_user_version();
6267         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6268         # load new efi bios files on migration. So this hack is required to allow
6269         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6270         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6271         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6272     }
6273
6274     return ($use_old_bios_files, $machine_type);
6275 }
6276
6277 sub lspci {
6278
6279     my $devices = {};
6280
6281     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6282             my (undef, $id, $function) = @_;
6283             my $res = { id => $id, function => $function};
6284             push @{$devices->{$id}}, $res;
6285     });
6286
6287     return $devices;
6288 }
6289
6290 sub vm_iothreads_list {
6291     my ($vmid) = @_;
6292
6293     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6294
6295     my $iothreads = {};
6296     foreach my $iothread (@$res) {
6297         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6298     }
6299
6300     return $iothreads;
6301 }
6302
6303 sub scsihw_infos {
6304     my ($conf, $drive) = @_;
6305
6306     my $maxdev = 0;
6307
6308     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6309         $maxdev = 7;
6310     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6311         $maxdev = 1;
6312     } else {
6313         $maxdev = 256;
6314     }
6315
6316     my $controller = int($drive->{index} / $maxdev);
6317     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6318
6319     return ($maxdev, $controller, $controller_prefix);
6320 }
6321
6322 sub add_hyperv_enlighments {
6323     my ($cpuFlags, $winversion, $machine_type, $kvmver, $nokvm, $bios, $gpu_passthrough) = @_;
6324
6325     return if $nokvm;
6326     return if $winversion < 6;
6327     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6328
6329     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6330
6331     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6332         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6333         push @$cpuFlags , 'hv_vapic';
6334         push @$cpuFlags , 'hv_time';
6335     } else {
6336         push @$cpuFlags , 'hv_spinlocks=0xffff';
6337     }
6338
6339     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6340         push @$cpuFlags , 'hv_reset';
6341         push @$cpuFlags , 'hv_vpindex';
6342         push @$cpuFlags , 'hv_runtime';
6343     }
6344
6345     if ($winversion >= 7) {
6346         push @$cpuFlags , 'hv_relaxed';
6347     }
6348 }
6349
6350 sub windows_version {
6351     my ($ostype) = @_;
6352
6353     return 0 if !$ostype;
6354
6355     my $winversion = 0;
6356
6357     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6358         $winversion = 5;
6359     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6360         $winversion = 6;
6361     } elsif ($ostype =~ m/^win(\d+)$/) {
6362         $winversion = $1;
6363     }
6364
6365     return $winversion;
6366 }
6367
6368 # bash completion helper
6369
6370 sub complete_backup_archives {
6371     my ($cmdname, $pname, $cvalue) = @_;
6372
6373     my $cfg = PVE::Storage::config();
6374
6375     my $storeid;
6376
6377     if ($cvalue =~ m/^([^:]+):/) {
6378         $storeid = $1;
6379     }
6380
6381     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6382
6383     my $res = [];
6384     foreach my $id (keys %$data) {
6385         foreach my $item (@{$data->{$id}}) {
6386             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6387             push @$res, $item->{volid} if defined($item->{volid});
6388         }
6389     }
6390
6391     return $res;
6392 }
6393
6394 my $complete_vmid_full = sub {
6395     my ($running) = @_;
6396
6397     my $idlist = vmstatus();
6398
6399     my $res = [];
6400
6401     foreach my $id (keys %$idlist) {
6402         my $d = $idlist->{$id};
6403         if (defined($running)) {
6404             next if $d->{template};
6405             next if $running && $d->{status} ne 'running';
6406             next if !$running && $d->{status} eq 'running';
6407         }
6408         push @$res, $id;
6409
6410     }
6411     return $res;
6412 };
6413
6414 sub complete_vmid {
6415     return &$complete_vmid_full();
6416 }
6417
6418 sub complete_vmid_stopped {
6419     return &$complete_vmid_full(0);
6420 }
6421
6422 sub complete_vmid_running {
6423     return &$complete_vmid_full(1);
6424 }
6425
6426 sub complete_storage {
6427
6428     my $cfg = PVE::Storage::config();
6429     my $ids = $cfg->{ids};
6430
6431     my $res = [];
6432     foreach my $sid (keys %$ids) {
6433         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6434         next if !$ids->{$sid}->{content}->{images};
6435         push @$res, $sid;
6436     }
6437
6438     return $res;
6439 }
6440
6441 sub nbd_stop {
6442     my ($vmid) = @_;
6443
6444     vm_mon_cmd($vmid, 'nbd-server-stop');
6445 }
6446
6447 1;