PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42 my $OVMF_IMG = '/usr/share/kvm/OVMF-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 # Note about locking: we use flock on the config file protect
  49 # against concurent actions.
  50 # Aditionaly, we have a 'lock' setting in the config file. This
  51 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  52 # allowed when such lock is set. But you can ignore this kind of
  53 # lock with the --skiplock flag.
  54
  55 cfs_register_file('/qemu-server/',
  56                   \&parse_vm_config,
  57                   \&write_vm_config);
  58
  59 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  60     description => "Some command save/restore state from this location.",
  61     type => 'string',
  62     maxLength => 128,
  63     optional => 1,
  64 });
  65
  66 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  67     description => "The name of the snapshot.",
  68     type => 'string', format => 'pve-configid',
  69     maxLength => 40,
  70 });
  71
  72 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  73     type => 'string',
  74     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  75     description => "The drive's backing file's data format.",
  76     optional => 1,
  77 });
  78
  79 #no warnings 'redefine';
  80
  81 sub cgroups_write {
  82    my ($controller, $vmid, $option, $value) = @_;
  83
  84    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  85    PVE::ProcFSTools::write_proc_entry($path, $value);
  86
  87 }
  88
  89 my $nodename = PVE::INotify::nodename();
  90
  91 mkdir "/etc/pve/nodes/$nodename";
  92 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  93 mkdir $confdir;
  94
  95 my $var_run_tmpdir = "/var/run/qemu-server";
  96 mkdir $var_run_tmpdir;
  97
  98 my $lock_dir = "/var/lock/qemu-server";
  99 mkdir $lock_dir;
 100
 101 my $pcisysfs = "/sys/bus/pci";
 102
 103 my $cpu_vendor_list = {
 104     # Intel CPUs
 105     486 => 'GenuineIntel',
 106     pentium => 'GenuineIntel',
 107     pentium2  => 'GenuineIntel',
 108     pentium3  => 'GenuineIntel',
 109     coreduo => 'GenuineIntel',
 110     core2duo => 'GenuineIntel',
 111     Conroe  => 'GenuineIntel',
 112     Penryn  => 'GenuineIntel',
 113     Nehalem  => 'GenuineIntel',
 114     Westmere => 'GenuineIntel',
 115     SandyBridge => 'GenuineIntel',
 116     IvyBridge => 'GenuineIntel',
 117     Haswell => 'GenuineIntel',
 118     'Haswell-noTSX' => 'GenuineIntel',
 119     Broadwell => 'GenuineIntel',
 120     'Broadwell-noTSX' => 'GenuineIntel',
 121     'Skylake-Client' => 'GenuineIntel',
 122
 123     # AMD CPUs
 124     athlon => 'AuthenticAMD',
 125     phenom  => 'AuthenticAMD',
 126     Opteron_G1  => 'AuthenticAMD',
 127     Opteron_G2  => 'AuthenticAMD',
 128     Opteron_G3  => 'AuthenticAMD',
 129     Opteron_G4  => 'AuthenticAMD',
 130     Opteron_G5  => 'AuthenticAMD',
 131
 132     # generic types, use vendor from host node
 133     host => 'default',
 134     kvm32 => 'default',
 135     kvm64 => 'default',
 136     qemu32 => 'default',
 137     qemu64 => 'default',
 138 };
 139
 140 my $cpu_fmt = {
 141     cputype => {
 142         description => "Emulated CPU type.",
 143         type => 'string',
 144         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 145         default => 'kvm64',
 146         default_key => 1,
 147     },
 148     hidden => {
 149         description => "Do not identify as a KVM virtual machine.",
 150         type => 'boolean',
 151         optional => 1,
 152         default => 0
 153     },
 154 };
 155
 156 my $watchdog_fmt = {
 157     model => {
 158         default_key => 1,
 159         type => 'string',
 160         enum => [qw(i6300esb ib700)],
 161         description => "Watchdog type to emulate.",
 162         default => 'i6300esb',
 163         optional => 1,
 164     },
 165     action => {
 166         type => 'string',
 167         enum => [qw(reset shutdown poweroff pause debug none)],
 168         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 169         optional => 1,
 170     },
 171 };
 172 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 173
 174 my $confdesc = {
 175     onboot => {
 176         optional => 1,
 177         type => 'boolean',
 178         description => "Specifies whether a VM will be started during system bootup.",
 179         default => 0,
 180     },
 181     autostart => {
 182         optional => 1,
 183         type => 'boolean',
 184         description => "Automatic restart after crash (currently ignored).",
 185         default => 0,
 186     },
 187     hotplug => {
 188         optional => 1,
 189         type => 'string', format => 'pve-hotplug-features',
 190         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 191         default => 'network,disk,usb',
 192     },
 193     reboot => {
 194         optional => 1,
 195         type => 'boolean',
 196         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 197         default => 1,
 198     },
 199     lock => {
 200         optional => 1,
 201         type => 'string',
 202         description => "Lock/unlock the VM.",
 203         enum => [qw(migrate backup snapshot rollback)],
 204     },
 205     cpulimit => {
 206         optional => 1,
 207         type => 'number',
 208         description => "Limit of CPU usage.",
 209         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 210         minimum => 0,
 211         maximum => 128,
 212         default => 0,
 213     },
 214     cpuunits => {
 215         optional => 1,
 216         type => 'integer',
 217         description => "CPU weight for a VM.",
 218         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.\n\nNOTE: You can disable fair-scheduler configuration by setting this to 0.",
 219         minimum => 0,
 220         maximum => 500000,
 221         default => 1024,
 222     },
 223     memory => {
 224         optional => 1,
 225         type => 'integer',
 226         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 227         minimum => 16,
 228         default => 512,
 229     },
 230     balloon => {
 231         optional => 1,
 232         type => 'integer',
 233         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 234         minimum => 0,
 235     },
 236     shares => {
 237         optional => 1,
 238         type => 'integer',
 239         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 240         minimum => 0,
 241         maximum => 50000,
 242         default => 1000,
 243     },
 244     keyboard => {
 245         optional => 1,
 246         type => 'string',
 247         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 248         enum => PVE::Tools::kvmkeymaplist(),
 249         default => 'en-us',
 250     },
 251     name => {
 252         optional => 1,
 253         type => 'string', format => 'dns-name',
 254         description => "Set a name for the VM. Only used on the configuration web interface.",
 255     },
 256     scsihw => {
 257         optional => 1,
 258         type => 'string',
 259         description => "SCSI controller model",
 260         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 261         default => 'lsi',
 262     },
 263     description => {
 264         optional => 1,
 265         type => 'string',
 266         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 267     },
 268     ostype => {
 269         optional => 1,
 270         type => 'string',
 271         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 272         description => "Specify guest operating system.",
 273         verbose_description => <<EODESC,
 274 Specify guest operating system. This is used to enable special
 275 optimization/features for specific operating systems:
 276
 277 [horizontal]
 278 other;; unspecified OS
 279 wxp;; Microsoft Windows XP
 280 w2k;; Microsoft Windows 2000
 281 w2k3;; Microsoft Windows 2003
 282 w2k8;; Microsoft Windows 2008
 283 wvista;; Microsoft Windows Vista
 284 win7;; Microsoft Windows 7
 285 win8;; Microsoft Windows 8/2012
 286 l24;; Linux 2.4 Kernel
 287 l26;; Linux 2.6/3.X Kernel
 288 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 289 EODESC
 290     },
 291     boot => {
 292         optional => 1,
 293         type => 'string',
 294         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 295         pattern => '[acdn]{1,4}',
 296         default => 'cdn',
 297     },
 298     bootdisk => {
 299         optional => 1,
 300         type => 'string', format => 'pve-qm-bootdisk',
 301         description => "Enable booting from specified disk.",
 302         pattern => '(ide|sata|scsi|virtio)\d+',
 303     },
 304     smp => {
 305         optional => 1,
 306         type => 'integer',
 307         description => "The number of CPUs. Please use option -sockets instead.",
 308         minimum => 1,
 309         default => 1,
 310     },
 311     sockets => {
 312         optional => 1,
 313         type => 'integer',
 314         description => "The number of CPU sockets.",
 315         minimum => 1,
 316         default => 1,
 317     },
 318     cores => {
 319         optional => 1,
 320         type => 'integer',
 321         description => "The number of cores per socket.",
 322         minimum => 1,
 323         default => 1,
 324     },
 325     numa => {
 326         optional => 1,
 327         type => 'boolean',
 328         description => "Enable/disable NUMA.",
 329         default => 0,
 330     },
 331     hugepages => {
 332         optional => 1,
 333         type => 'string',
 334         description => "Enable/disable hugepages memory.",
 335         enum => [qw(any 2 1024)],
 336     },
 337     vcpus => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "Number of hotplugged vcpus.",
 341         minimum => 1,
 342         default => 0,
 343     },
 344     acpi => {
 345         optional => 1,
 346         type => 'boolean',
 347         description => "Enable/disable ACPI.",
 348         default => 1,
 349     },
 350     agent => {
 351         optional => 1,
 352         type => 'boolean',
 353         description => "Enable/disable Qemu GuestAgent.",
 354         default => 0,
 355     },
 356     kvm => {
 357         optional => 1,
 358         type => 'boolean',
 359         description => "Enable/disable KVM hardware virtualization.",
 360         default => 1,
 361     },
 362     tdf => {
 363         optional => 1,
 364         type => 'boolean',
 365         description => "Enable/disable time drift fix.",
 366         default => 0,
 367     },
 368     localtime => {
 369         optional => 1,
 370         type => 'boolean',
 371         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 372     },
 373     freeze => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 377     },
 378     vga => {
 379         optional => 1,
 380         type => 'string',
 381         description => "Select the VGA type.",
 382         verbose_description => "Select the VGA type. If you want to use high resolution" .
 383             " modes (>= 1280x1024x16) then you should use the options " .
 384             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 385             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 386             "display sever. For win* OS you can select how many independent " .
 387             "displays you want, Linux guests can add displays them self. " .
 388             "You can also run without any graphic card, using a serial device" .
 389             " as terminal.",
 390         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 391     },
 392     watchdog => {
 393         optional => 1,
 394         type => 'string', format => 'pve-qm-watchdog',
 395         description => "Create a virtual hardware watchdog device.",
 396         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 397             " (by a guest action), the watchdog must be periodically polled " .
 398             "by an agent inside the guest or else the watchdog will reset " .
 399             "the guest (or execute the respective action specified)",
 400     },
 401     startdate => {
 402         optional => 1,
 403         type => 'string',
 404         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 405         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 406         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 407         default => 'now',
 408     },
 409     startup =>  get_standard_option('pve-startup-order'),
 410     template => {
 411         optional => 1,
 412         type => 'boolean',
 413         description => "Enable/disable Template.",
 414         default => 0,
 415     },
 416     args => {
 417         optional => 1,
 418         type => 'string',
 419         description => "Arbitrary arguments passed to kvm.",
 420         verbose_description => <<EODESCR,
 421 Arbitrary arguments passed to kvm, for example:
 422
 423 args: -no-reboot -no-hpet
 424
 425 NOTE: this option is for experts only.
 426 EODESCR
 427     },
 428     tablet => {
 429         optional => 1,
 430         type => 'boolean',
 431         default => 1,
 432         description => "Enable/disable the USB tablet device.",
 433         verbose_description => "Enable/disable the USB tablet device. This device is " .
 434             "usually needed to allow absolute mouse positioning with VNC. " .
 435             "Else the mouse runs out of sync with normal VNC clients. " .
 436             "If you're running lots of console-only guests on one host, " .
 437             "you may consider disabling this to save some context switches. " .
 438             "This is turned off by default if you use spice (-vga=qxl).",
 439     },
 440     migrate_speed => {
 441         optional => 1,
 442         type => 'integer',
 443         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 444         minimum => 0,
 445         default => 0,
 446     },
 447     migrate_downtime => {
 448         optional => 1,
 449         type => 'number',
 450         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 451         minimum => 0,
 452         default => 0.1,
 453     },
 454     cdrom => {
 455         optional => 1,
 456         type => 'string', format => 'pve-qm-ide',
 457         typetext => '<volume>',
 458         description => "This is an alias for option -ide2",
 459     },
 460     cpu => {
 461         optional => 1,
 462         description => "Emulated CPU type.",
 463         type => 'string',
 464         format => $cpu_fmt,
 465     },
 466     parent => get_standard_option('pve-snapshot-name', {
 467         optional => 1,
 468         description => "Parent snapshot name. This is used internally, and should not be modified.",
 469     }),
 470     snaptime => {
 471         optional => 1,
 472         description => "Timestamp for snapshots.",
 473         type => 'integer',
 474         minimum => 0,
 475     },
 476     vmstate => {
 477         optional => 1,
 478         type => 'string', format => 'pve-volume-id',
 479         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 480     },
 481     machine => {
 482         description => "Specific the Qemu machine type.",
 483         type => 'string',
 484         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 485         maxLength => 40,
 486         optional => 1,
 487     },
 488     smbios1 => {
 489         description => "Specify SMBIOS type 1 fields.",
 490         type => 'string', format => 'pve-qm-smbios1',
 491         maxLength => 256,
 492         optional => 1,
 493     },
 494     protection => {
 495         optional => 1,
 496         type => 'boolean',
 497         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 498         default => 0,
 499     },
 500     bios => {
 501         optional => 1,
 502         type => 'string',
 503         enum => [ qw(seabios ovmf) ],
 504         description => "Select BIOS implementation.",
 505         default => 'seabios',
 506     },
 507 };
 508
 509 # what about other qemu settings ?
 510 #cpu => 'string',
 511 #machine => 'string',
 512 #fda => 'file',
 513 #fdb => 'file',
 514 #mtdblock => 'file',
 515 #sd => 'file',
 516 #pflash => 'file',
 517 #snapshot => 'bool',
 518 #bootp => 'file',
 519 ##tftp => 'dir',
 520 ##smb => 'dir',
 521 #kernel => 'file',
 522 #append => 'string',
 523 #initrd => 'file',
 524 ##soundhw => 'string',
 525
 526 while (my ($k, $v) = each %$confdesc) {
 527     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 528 }
 529
 530 my $MAX_IDE_DISKS = 4;
 531 my $MAX_SCSI_DISKS = 14;
 532 my $MAX_VIRTIO_DISKS = 16;
 533 my $MAX_SATA_DISKS = 6;
 534 my $MAX_USB_DEVICES = 5;
 535 my $MAX_NETS = 32;
 536 my $MAX_UNUSED_DISKS = 8;
 537 my $MAX_HOSTPCI_DEVICES = 4;
 538 my $MAX_SERIAL_PORTS = 4;
 539 my $MAX_PARALLEL_PORTS = 3;
 540 my $MAX_NUMA = 8;
 541
 542 my $numa_fmt = {
 543     cpus => {
 544         type => "string",
 545         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 546         description => "CPUs accessing this NUMA node.",
 547         format_description => "id[-id];...",
 548     },
 549     memory => {
 550         type => "number",
 551         description => "Amount of memory this NUMA node provides.",
 552         optional => 1,
 553     },
 554     hostnodes => {
 555         type => "string",
 556         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 557         description => "Host NUMA nodes to use.",
 558         format_description => "id[-id];...",
 559         optional => 1,
 560     },
 561     policy => {
 562         type => 'string',
 563         enum => [qw(preferred bind interleave)],
 564         description => "NUMA allocation policy.",
 565         optional => 1,
 566     },
 567 };
 568 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 569 my $numadesc = {
 570     optional => 1,
 571     type => 'string', format => $numa_fmt,
 572     description => "NUMA topology.",
 573 };
 574 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 575
 576 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 577     $confdesc->{"numa$i"} = $numadesc;
 578 }
 579
 580 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 581                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 582                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 583 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 584
 585 my $net_fmt_bridge_descr = <<__EOD__;
 586 Bridge to attach the network device to. The Proxmox VE standard bridge
 587 is called 'vmbr0'.
 588
 589 If you do not specify a bridge, we create a kvm user (NATed) network
 590 device, which provides DHCP and DNS services. The following addresses
 591 are used:
 592
 593  10.0.2.2   Gateway
 594  10.0.2.3   DNS Server
 595  10.0.2.4   SMB Server
 596
 597 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 598 __EOD__
 599
 600 my $net_fmt = {
 601     macaddr => {
 602         type => 'string',
 603         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 604         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 605         format_description => "XX:XX:XX:XX:XX:XX",
 606         optional => 1,
 607     },
 608     model => {
 609         type => 'string',
 610         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 611         enum => $nic_model_list,
 612         default_key => 1,
 613     },
 614     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 615     bridge => {
 616         type => 'string',
 617         description => $net_fmt_bridge_descr,
 618         format_description => 'bridge',
 619         optional => 1,
 620     },
 621     queues => {
 622         type => 'integer',
 623         minimum => 0, maximum => 16,
 624         description => 'Number of packet queues to be used on the device.',
 625         optional => 1,
 626     },
 627     rate => {
 628         type => 'number',
 629         minimum => 0,
 630         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 631         optional => 1,
 632     },
 633     tag => {
 634         type => 'integer',
 635         minimum => 1, maximum => 4094,
 636         description => 'VLAN tag to apply to packets on this interface.',
 637         optional => 1,
 638     },
 639     trunks => {
 640         type => 'string',
 641         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 642         description => 'VLAN trunks to pass through this interface.',
 643         format_description => 'vlanid[;vlanid...]',
 644         optional => 1,
 645     },
 646     firewall => {
 647         type => 'boolean',
 648         description => 'Whether this interface should be protected by the firewall.',
 649         optional => 1,
 650     },
 651     link_down => {
 652         type => 'boolean',
 653         description => 'Whether this interface should be disconnected (like pulling the plug).',
 654         optional => 1,
 655     },
 656 };
 657
 658 my $netdesc = {
 659     optional => 1,
 660     type => 'string', format => $net_fmt,
 661     description => "Specify network devices.",
 662 };
 663
 664 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 665
 666 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 667     $confdesc->{"net$i"} = $netdesc;
 668 }
 669
 670 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 671 sub verify_volume_id_or_qm_path {
 672     my ($volid, $noerr) = @_;
 673
 674     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 675         return $volid;
 676     }
 677
 678     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 679     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 680     if ($@) {
 681         return undef if $noerr;
 682         die $@;
 683     }
 684     return $volid;
 685 }
 686
 687 my $drivename_hash;
 688
 689 my %drivedesc_base = (
 690     volume => { alias => 'file' },
 691     file => {
 692         type => 'string',
 693         format => 'pve-volume-id-or-qm-path',
 694         default_key => 1,
 695         format_description => 'volume',
 696         description => "The drive's backing volume.",
 697     },
 698     media => {
 699         type => 'string',
 700         enum => [qw(cdrom disk)],
 701         description => "The drive's media type.",
 702         default => 'disk',
 703         optional => 1
 704     },
 705     cyls => {
 706         type => 'integer',
 707         description => "Force the drive's physical geometry to have a specific cylinder count.",
 708         optional => 1
 709     },
 710     heads => {
 711         type => 'integer',
 712         description => "Force the drive's physical geometry to have a specific head count.",
 713         optional => 1
 714     },
 715     secs => {
 716         type => 'integer',
 717         description => "Force the drive's physical geometry to have a specific sector count.",
 718         optional => 1
 719     },
 720     trans => {
 721         type => 'string',
 722         enum => [qw(none lba auto)],
 723         description => "Force disk geometry bios translation mode.",
 724         optional => 1,
 725     },
 726     snapshot => {
 727         type => 'boolean',
 728         description => "Whether the drive should be included when making snapshots.",
 729         optional => 1,
 730     },
 731     cache => {
 732         type => 'string',
 733         enum => [qw(none writethrough writeback unsafe directsync)],
 734         description => "The drive's cache mode",
 735         optional => 1,
 736     },
 737     format => get_standard_option('pve-qm-image-format'),
 738     size => {
 739         type => 'string',
 740         format => 'disk-size',
 741         format_description => 'DiskSize',
 742         description => "Disk size. This is purely informational and has no effect.",
 743         optional => 1,
 744     },
 745     backup => {
 746         type => 'boolean',
 747         description => "Whether the drive should be included when making backups.",
 748         optional => 1,
 749     },
 750     rerror => {
 751         type => 'string',
 752         enum => [qw(ignore report stop)],
 753         description => 'Read error action.',
 754         optional => 1,
 755     },
 756     werror => {
 757         type => 'string',
 758         enum => [qw(enospc ignore report stop)],
 759         description => 'Write error action.',
 760         optional => 1,
 761     },
 762     aio => {
 763         type => 'string',
 764         enum => [qw(native threads)],
 765         description => 'AIO type to use.',
 766         optional => 1,
 767     },
 768     discard => {
 769         type => 'string',
 770         enum => [qw(ignore on)],
 771         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 772         optional => 1,
 773     },
 774     detect_zeroes => {
 775         type => 'boolean',
 776         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 777         optional => 1,
 778     },
 779     serial => {
 780         type => 'string',
 781         format => 'urlencoded',
 782         format_description => 'serial',
 783         maxLength => 20*3, # *3 since it's %xx url enoded
 784         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 785         optional => 1,
 786     }
 787 );
 788
 789 my %iothread_fmt = ( iothread => {
 790         type => 'boolean',
 791         description => "Whether to use iothreads for this drive",
 792         optional => 1,
 793 });
 794
 795 my %model_fmt = (
 796     model => {
 797         type => 'string',
 798         format => 'urlencoded',
 799         format_description => 'model',
 800         maxLength => 40*3, # *3 since it's %xx url enoded
 801         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 802         optional => 1,
 803     },
 804 );
 805
 806 my %queues_fmt = (
 807     queues => {
 808         type => 'integer',
 809         description => "Number of queues.",
 810         minimum => 2,
 811         optional => 1
 812     }
 813 );
 814
 815 my %scsiblock_fmt = (
 816     scsiblock => {
 817         type => 'boolean',
 818         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 819         optional => 1,
 820         default => 0,
 821     },
 822 );
 823
 824 my $add_throttle_desc = sub {
 825     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 826     my $d = {
 827         type => $type,
 828         format_description => $unit,
 829         description => "Maximum $what in $longunit.",
 830         optional => 1,
 831     };
 832     $d->{minimum} = $minimum if defined($minimum);
 833     $drivedesc_base{$key} = $d;
 834 };
 835 # throughput: (leaky bucket)
 836 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 837 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 838 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 839 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 840 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 841 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 842 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 843 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 844 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 845
 846 # pools: (pool of IO before throttling starts taking effect)
 847 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 848 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 849 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 850 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 851 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 852 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 853
 854 # burst lengths
 855 $add_throttle_desc->('bps_max_length',  'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 856 $add_throttle_desc->('bps_rd_length',   'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 857 $add_throttle_desc->('bps_wr_length',   'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 858 $add_throttle_desc->('iops_max_length', 'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 859 $add_throttle_desc->('iops_rd_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 860 $add_throttle_desc->('iops_wr_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 861
 862 my $ide_fmt = {
 863     %drivedesc_base,
 864     %model_fmt,
 865 };
 866 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 867
 868 my $idedesc = {
 869     optional => 1,
 870     type => 'string', format => $ide_fmt,
 871     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 872 };
 873 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 874
 875 my $scsi_fmt = {
 876     %drivedesc_base,
 877     %iothread_fmt,
 878     %queues_fmt,
 879     %scsiblock_fmt,
 880 };
 881 my $scsidesc = {
 882     optional => 1,
 883     type => 'string', format => $scsi_fmt,
 884     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 885 };
 886 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 887
 888 my $sata_fmt = {
 889     %drivedesc_base,
 890 };
 891 my $satadesc = {
 892     optional => 1,
 893     type => 'string', format => $sata_fmt,
 894     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 895 };
 896 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 897
 898 my $virtio_fmt = {
 899     %drivedesc_base,
 900     %iothread_fmt,
 901 };
 902 my $virtiodesc = {
 903     optional => 1,
 904     type => 'string', format => $virtio_fmt,
 905     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 906 };
 907 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 908
 909 my $alldrive_fmt = {
 910     %drivedesc_base,
 911     %iothread_fmt,
 912     %model_fmt,
 913     %queues_fmt,
 914     %scsiblock_fmt,
 915 };
 916
 917 my $efidisk_fmt = {
 918     volume => { alias => 'file' },
 919     file => {
 920         type => 'string',
 921         format => 'pve-volume-id-or-qm-path',
 922         default_key => 1,
 923         format_description => 'volume',
 924         description => "The drive's backing volume.",
 925     },
 926     format => get_standard_option('pve-qm-image-format'),
 927     size => {
 928         type => 'string',
 929         format => 'disk-size',
 930         format_description => 'DiskSize',
 931         description => "Disk size. This is purely informational and has no effect.",
 932         optional => 1,
 933     },
 934 };
 935
 936 my $efidisk_desc = {
 937     optional => 1,
 938     type => 'string', format => $efidisk_fmt,
 939     description => "Configure a Disk for storing EFI vars",
 940 };
 941
 942 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 943
 944 my $usb_fmt = {
 945     host => {
 946         default_key => 1,
 947         type => 'string', format => 'pve-qm-usb-device',
 948         format_description => 'HOSTUSBDEVICE|spice',
 949         description => <<EODESCR,
 950 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 951
 952  'bus-port(.port)*' (decimal numbers) or
 953  'vendor_id:product_id' (hexadeciaml numbers) or
 954  'spice'
 955
 956 You can use the 'lsusb -t' command to list existing usb devices.
 957
 958 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 959
 960 The value 'spice' can be used to add a usb redirection devices for spice.
 961 EODESCR
 962     },
 963     usb3 => {
 964         optional => 1,
 965         type => 'boolean',
 966         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 967         default => 0,
 968     },
 969 };
 970
 971 my $usbdesc = {
 972     optional => 1,
 973     type => 'string', format => $usb_fmt,
 974     description => "Configure an USB device (n is 0 to 4).",
 975 };
 976 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 977
 978 # NOTE: the match-groups of this regex are used in parse_hostpci
 979 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 980 my $hostpci_fmt = {
 981     host => {
 982         default_key => 1,
 983         type => 'string',
 984         pattern => qr/$PCIRE(;$PCIRE)*/,
 985         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
 986         description => <<EODESCR,
 987 Host PCI device pass through. The PCI ID of a host's PCI device or a list
 988 of PCI virtual functions of the host. HOSTPCIID syntax is:
 989
 990 'bus:dev.func' (hexadecimal numbers)
 991
 992 You can us the 'lspci' command to list existing PCI devices.
 993 EODESCR
 994     },
 995     rombar => {
 996         type => 'boolean',
 997         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
 998         optional => 1,
 999         default => 1,
1000     },
1001     romfile => {
1002         type => 'string',
1003         pattern => '[^,;]+',
1004         format_description => 'string',
1005         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1006         optional => 1,
1007     },
1008     pcie => {
1009         type => 'boolean',
1010         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1011         optional => 1,
1012         default => 0,
1013     },
1014     'x-vga' => {
1015         type => 'boolean',
1016         description =>  "Enable vfio-vga device support.",
1017         optional => 1,
1018         default => 0,
1019     },
1020 };
1021 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1022
1023 my $hostpcidesc = {
1024         optional => 1,
1025         type => 'string', format => 'pve-qm-hostpci',
1026         description => "Map host PCI devices into guest.",
1027         verbose_description =>  <<EODESCR,
1028 Map host PCI devices into guest.
1029
1030 NOTE: This option allows direct access to host hardware. So it is no longer
1031 possible to migrate such machines - use with special care.
1032
1033 CAUTION: Experimental! User reported problems with this option.
1034 EODESCR
1035 };
1036 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1037
1038 my $serialdesc = {
1039         optional => 1,
1040         type => 'string',
1041         pattern => '(/dev/.+|socket)',
1042         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1043         verbose_description =>  <<EODESCR,
1044 Create a serial device inside the VM (n is 0 to 3), and pass through a
1045 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1046 host side (use 'qm terminal' to open a terminal connection).
1047
1048 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1049
1050 CAUTION: Experimental! User reported problems with this option.
1051 EODESCR
1052 };
1053
1054 my $paralleldesc= {
1055         optional => 1,
1056         type => 'string',
1057         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1058         description =>  "Map host parallel devices (n is 0 to 2).",
1059         verbose_description =>  <<EODESCR,
1060 Map host parallel devices (n is 0 to 2).
1061
1062 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1063
1064 CAUTION: Experimental! User reported problems with this option.
1065 EODESCR
1066 };
1067
1068 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1069     $confdesc->{"parallel$i"} = $paralleldesc;
1070 }
1071
1072 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1073     $confdesc->{"serial$i"} = $serialdesc;
1074 }
1075
1076 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1077     $confdesc->{"hostpci$i"} = $hostpcidesc;
1078 }
1079
1080 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1081     $drivename_hash->{"ide$i"} = 1;
1082     $confdesc->{"ide$i"} = $idedesc;
1083 }
1084
1085 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1086     $drivename_hash->{"sata$i"} = 1;
1087     $confdesc->{"sata$i"} = $satadesc;
1088 }
1089
1090 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1091     $drivename_hash->{"scsi$i"} = 1;
1092     $confdesc->{"scsi$i"} = $scsidesc ;
1093 }
1094
1095 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1096     $drivename_hash->{"virtio$i"} = 1;
1097     $confdesc->{"virtio$i"} = $virtiodesc;
1098 }
1099
1100 $drivename_hash->{efidisk0} = 1;
1101 $confdesc->{efidisk0} = $efidisk_desc;
1102
1103 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1104     $confdesc->{"usb$i"} = $usbdesc;
1105 }
1106
1107 my $unuseddesc = {
1108     optional => 1,
1109     type => 'string', format => 'pve-volume-id',
1110     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1111 };
1112
1113 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1114     $confdesc->{"unused$i"} = $unuseddesc;
1115 }
1116
1117 my $kvm_api_version = 0;
1118
1119 sub kvm_version {
1120
1121     return $kvm_api_version if $kvm_api_version;
1122
1123     my $fh = IO::File->new("</dev/kvm") ||
1124         return 0;
1125
1126     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1127         $kvm_api_version = $v;
1128     }
1129
1130     $fh->close();
1131
1132     return  $kvm_api_version;
1133 }
1134
1135 my $kvm_user_version;
1136
1137 sub kvm_user_version {
1138
1139     return $kvm_user_version if $kvm_user_version;
1140
1141     $kvm_user_version = 'unknown';
1142
1143     my $code = sub {
1144         my $line = shift;
1145         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1146             $kvm_user_version = $2;
1147         }
1148     };
1149
1150     eval { run_command("kvm -version", outfunc => $code); };
1151     warn $@ if $@;
1152
1153     return $kvm_user_version;
1154
1155 }
1156
1157 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1158
1159 sub valid_drive_names {
1160     # order is important - used to autoselect boot disk
1161     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1162             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1163             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1164             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1165             'efidisk0');
1166 }
1167
1168 sub is_valid_drivename {
1169     my $dev = shift;
1170
1171     return defined($drivename_hash->{$dev});
1172 }
1173
1174 sub option_exists {
1175     my $key = shift;
1176     return defined($confdesc->{$key});
1177 }
1178
1179 sub nic_models {
1180     return $nic_model_list;
1181 }
1182
1183 sub os_list_description {
1184
1185     return {
1186         other => 'Other',
1187         wxp => 'Windows XP',
1188         w2k => 'Windows 2000',
1189         w2k3 =>, 'Windows 2003',
1190         w2k8 => 'Windows 2008',
1191         wvista => 'Windows Vista',
1192         win7 => 'Windows 7',
1193         win8 => 'Windows 8/2012',
1194         win10 => 'Windows 10/2016',
1195         l24 => 'Linux 2.4',
1196         l26 => 'Linux 2.6',
1197     };
1198 }
1199
1200 my $cdrom_path;
1201
1202 sub get_cdrom_path {
1203
1204     return  $cdrom_path if $cdrom_path;
1205
1206     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1207     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1208     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1209 }
1210
1211 sub get_iso_path {
1212     my ($storecfg, $vmid, $cdrom) = @_;
1213
1214     if ($cdrom eq 'cdrom') {
1215         return get_cdrom_path();
1216     } elsif ($cdrom eq 'none') {
1217         return '';
1218     } elsif ($cdrom =~ m|^/|) {
1219         return $cdrom;
1220     } else {
1221         return PVE::Storage::path($storecfg, $cdrom);
1222     }
1223 }
1224
1225 # try to convert old style file names to volume IDs
1226 sub filename_to_volume_id {
1227     my ($vmid, $file, $media) = @_;
1228
1229     if (!($file eq 'none' || $file eq 'cdrom' ||
1230           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1231
1232         return undef if $file =~ m|/|;
1233
1234         if ($media && $media eq 'cdrom') {
1235             $file = "local:iso/$file";
1236         } else {
1237             $file = "local:$vmid/$file";
1238         }
1239     }
1240
1241     return $file;
1242 }
1243
1244 sub verify_media_type {
1245     my ($opt, $vtype, $media) = @_;
1246
1247     return if !$media;
1248
1249     my $etype;
1250     if ($media eq 'disk') {
1251         $etype = 'images';
1252     } elsif ($media eq 'cdrom') {
1253         $etype = 'iso';
1254     } else {
1255         die "internal error";
1256     }
1257
1258     return if ($vtype eq $etype);
1259
1260     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1261 }
1262
1263 sub cleanup_drive_path {
1264     my ($opt, $storecfg, $drive) = @_;
1265
1266     # try to convert filesystem paths to volume IDs
1267
1268     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1269         ($drive->{file} !~ m|^/dev/.+|) &&
1270         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1271         ($drive->{file} !~ m/^\d+$/)) {
1272         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1273         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1274         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1275         verify_media_type($opt, $vtype, $drive->{media});
1276         $drive->{file} = $volid;
1277     }
1278
1279     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1280 }
1281
1282 sub parse_hotplug_features {
1283     my ($data) = @_;
1284
1285     my $res = {};
1286
1287     return $res if $data eq '0';
1288
1289     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1290
1291     foreach my $feature (PVE::Tools::split_list($data)) {
1292         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1293             $res->{$1} = 1;
1294         } else {
1295             die "invalid hotplug feature '$feature'\n";
1296         }
1297     }
1298     return $res;
1299 }
1300
1301 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1302 sub pve_verify_hotplug_features {
1303     my ($value, $noerr) = @_;
1304
1305     return $value if parse_hotplug_features($value);
1306
1307     return undef if $noerr;
1308
1309     die "unable to parse hotplug option\n";
1310 }
1311
1312 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1313 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1314 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1315 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1316 #        [,iothread=on][,serial=serial][,model=model]
1317
1318 sub parse_drive {
1319     my ($key, $data) = @_;
1320
1321     my ($interface, $index);
1322
1323     if ($key =~ m/^([^\d]+)(\d+)$/) {
1324         $interface = $1;
1325         $index = $2;
1326     } else {
1327         return undef;
1328     }
1329
1330     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1331                                      : $confdesc->{$key}->{format};
1332     if (!$desc) {
1333         warn "invalid drive key: $key\n";
1334         return undef;
1335     }
1336     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1337     return undef if !$res;
1338     $res->{interface} = $interface;
1339     $res->{index} = $index;
1340
1341     my $error = 0;
1342     foreach my $opt (qw(bps bps_rd bps_wr)) {
1343         if (my $bps = defined(delete $res->{$opt})) {
1344             if (defined($res->{"m$opt"})) {
1345                 warn "both $opt and m$opt specified\n";
1346                 ++$error;
1347                 next;
1348             }
1349             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1350         }
1351     }
1352
1353     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1354     for my $requirement (
1355         [bps_max_length => 'mbps_max'],
1356         [bps_rd_max_length => 'mbps_rd_max'],
1357         [bps_wr_max_length => 'mbps_wr_max'],
1358         [iops_max_length => 'iops_max'],
1359         [iops_rd_max_length => 'iops_rd_max'],
1360         [iops_wr_max_length => 'iops_wr_max']) {
1361         my ($option, $requires) = @$requirement;
1362         if ($res->{$option} && !$res->{$requires}) {
1363             warn "$option requires $requires\n";
1364             ++$error;
1365         }
1366     }
1367
1368     return undef if $error;
1369
1370     return undef if $res->{mbps_rd} && $res->{mbps};
1371     return undef if $res->{mbps_wr} && $res->{mbps};
1372     return undef if $res->{iops_rd} && $res->{iops};
1373     return undef if $res->{iops_wr} && $res->{iops};
1374
1375     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1376         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1377         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1378         return undef if $res->{interface} eq 'virtio';
1379     }
1380
1381     if (my $size = $res->{size}) {
1382         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1383     }
1384
1385     return $res;
1386 }
1387
1388 sub print_drive {
1389     my ($vmid, $drive) = @_;
1390     my $data = { %$drive };
1391     delete $data->{$_} for qw(index interface);
1392     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1393 }
1394
1395 sub scsi_inquiry {
1396     my($fh, $noerr) = @_;
1397
1398     my $SG_IO = 0x2285;
1399     my $SG_GET_VERSION_NUM = 0x2282;
1400
1401     my $versionbuf = "\x00" x 8;
1402     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1403     if (!$ret) {
1404         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1405         return undef;
1406     }
1407     my $version = unpack("I", $versionbuf);
1408     if ($version < 30000) {
1409         die "scsi generic interface too old\n"  if !$noerr;
1410         return undef;
1411     }
1412
1413     my $buf = "\x00" x 36;
1414     my $sensebuf = "\x00" x 8;
1415     my $cmd = pack("C x3 C x1", 0x12, 36);
1416
1417     # see /usr/include/scsi/sg.h
1418     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1419
1420     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1421                       length($sensebuf), 0, length($buf), $buf,
1422                       $cmd, $sensebuf, 6000);
1423
1424     $ret = ioctl($fh, $SG_IO, $packet);
1425     if (!$ret) {
1426         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1427         return undef;
1428     }
1429
1430     my @res = unpack($sg_io_hdr_t, $packet);
1431     if ($res[17] || $res[18]) {
1432         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1433         return undef;
1434     }
1435
1436     my $res = {};
1437     (my $byte0, my $byte1, $res->{vendor},
1438      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1439
1440     $res->{removable} = $byte1 & 128 ? 1 : 0;
1441     $res->{type} = $byte0 & 31;
1442
1443     return $res;
1444 }
1445
1446 sub path_is_scsi {
1447     my ($path) = @_;
1448
1449     my $fh = IO::File->new("+<$path") || return undef;
1450     my $res = scsi_inquiry($fh, 1);
1451     close($fh);
1452
1453     return $res;
1454 }
1455
1456 sub machine_type_is_q35 {
1457     my ($conf) = @_;
1458
1459     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1460 }
1461
1462 sub print_tabletdevice_full {
1463     my ($conf) = @_;
1464
1465     my $q35 = machine_type_is_q35($conf);
1466
1467     # we use uhci for old VMs because tablet driver was buggy in older qemu
1468     my $usbbus = $q35 ? "ehci" : "uhci";
1469
1470     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1471 }
1472
1473 sub print_drivedevice_full {
1474     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1475
1476     my $device = '';
1477     my $maxdev = 0;
1478
1479     if ($drive->{interface} eq 'virtio') {
1480         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1481         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1482         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1483     } elsif ($drive->{interface} eq 'scsi') {
1484
1485         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1486         my $unit = $drive->{index} % $maxdev;
1487         my $devicetype = 'hd';
1488         my $path = '';
1489         if (drive_is_cdrom($drive)) {
1490             $devicetype = 'cd';
1491         } else {
1492             if ($drive->{file} =~ m|^/|) {
1493                 $path = $drive->{file};
1494                 if (my $info = path_is_scsi($path)) {
1495                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1496                         $devicetype = 'block';
1497                     } elsif ($info->{type} == 1) { # tape
1498                         $devicetype = 'generic';
1499                     }
1500                 }
1501             } else {
1502                  $path = PVE::Storage::path($storecfg, $drive->{file});
1503             }
1504
1505             if($path =~ m/^iscsi\:\/\//){
1506                 $devicetype = 'generic';
1507             }
1508         }
1509
1510         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1511             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1512         } else {
1513             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1514         }
1515
1516     } elsif ($drive->{interface} eq 'ide'){
1517         $maxdev = 2;
1518         my $controller = int($drive->{index} / $maxdev);
1519         my $unit = $drive->{index} % $maxdev;
1520         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1521
1522         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1523         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1524             $model = URI::Escape::uri_unescape($model);
1525             $device .= ",model=$model";
1526         }
1527     } elsif ($drive->{interface} eq 'sata'){
1528         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1529         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1530         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1531     } elsif ($drive->{interface} eq 'usb') {
1532         die "implement me";
1533         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1534     } else {
1535         die "unsupported interface type";
1536     }
1537
1538     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1539
1540     return $device;
1541 }
1542
1543 sub get_initiator_name {
1544     my $initiator;
1545
1546     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1547     while (defined(my $line = <$fh>)) {
1548         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1549         $initiator = $1;
1550         last;
1551     }
1552     $fh->close();
1553
1554     return $initiator;
1555 }
1556
1557 sub print_drive_full {
1558     my ($storecfg, $vmid, $drive) = @_;
1559
1560     my $path;
1561     my $volid = $drive->{file};
1562     my $format;
1563
1564     if (drive_is_cdrom($drive)) {
1565         $path = get_iso_path($storecfg, $vmid, $volid);
1566     } else {
1567         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1568         if ($storeid) {
1569             $path = PVE::Storage::path($storecfg, $volid);
1570             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1571             $format = qemu_img_format($scfg, $volname);
1572         } else {
1573             $path = $volid;
1574             $format = "raw";
1575         }
1576    }
1577
1578     my $opts = '';
1579     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard iops iops_rd iops_wr iops_max iops_rd_max iops_wr_max);
1580     foreach my $o (@qemu_drive_options) {
1581         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1582     }
1583     if (my $serial = $drive->{serial}) {
1584         $serial = URI::Escape::uri_unescape($serial);
1585         $opts .= ",serial=$serial";
1586     }
1587
1588     $opts .= ",format=$format" if $format && !$drive->{format};
1589
1590     foreach my $o (qw(bps bps_rd bps_wr)) {
1591         my $v = $drive->{"m$o"};
1592         $opts .= ",$o=" . int($v*1024*1024) if $v;
1593     }
1594
1595     my $cache_direct = 0;
1596
1597     if (my $cache = $drive->{cache}) {
1598         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1599     } elsif (!drive_is_cdrom($drive)) {
1600         $opts .= ",cache=none";
1601         $cache_direct = 1;
1602     }
1603
1604     # aio native works only with O_DIRECT
1605     if (!$drive->{aio}) {
1606         if($cache_direct) {
1607             $opts .= ",aio=native";
1608         } else {
1609             $opts .= ",aio=threads";
1610         }
1611     }
1612
1613     if (!drive_is_cdrom($drive)) {
1614         my $detectzeroes;
1615         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1616             $detectzeroes = 'off';
1617         } elsif ($drive->{discard}) {
1618             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1619         } else {
1620             # This used to be our default with discard not being specified:
1621             $detectzeroes = 'on';
1622         }
1623         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1624     }
1625
1626     my $pathinfo = $path ? "file=$path," : '';
1627
1628     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1629 }
1630
1631 sub print_netdevice_full {
1632     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1633
1634     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1635
1636     my $device = $net->{model};
1637     if ($net->{model} eq 'virtio') {
1638          $device = 'virtio-net-pci';
1639      };
1640
1641     my $pciaddr = print_pci_addr("$netid", $bridges);
1642     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1643     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1644         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1645         my $vectors = $net->{queues} * 2 + 2;
1646         $tmpstr .= ",vectors=$vectors,mq=on";
1647     }
1648     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1649
1650     if ($use_old_bios_files) {
1651         my $romfile;
1652         if ($device eq 'virtio-net-pci') {
1653             $romfile = 'pxe-virtio.rom';
1654         } elsif ($device eq 'e1000') {
1655             $romfile = 'pxe-e1000.rom';
1656         } elsif ($device eq 'ne2k') {
1657             $romfile = 'pxe-ne2k_pci.rom';
1658         } elsif ($device eq 'pcnet') {
1659             $romfile = 'pxe-pcnet.rom';
1660         } elsif ($device eq 'rtl8139') {
1661             $romfile = 'pxe-rtl8139.rom';
1662         }
1663         $tmpstr .= ",romfile=$romfile" if $romfile;
1664     }
1665
1666     return $tmpstr;
1667 }
1668
1669 sub print_netdev_full {
1670     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1671
1672     my $i = '';
1673     if ($netid =~ m/^net(\d+)$/) {
1674         $i = int($1);
1675     }
1676
1677     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1678
1679     my $ifname = "tap${vmid}i$i";
1680
1681     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1682     die "interface name '$ifname' is too long (max 15 character)\n"
1683         if length($ifname) >= 16;
1684
1685     my $vhostparam = '';
1686     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1687
1688     my $vmname = $conf->{name} || "vm$vmid";
1689
1690     my $netdev = "";
1691     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1692
1693     if ($net->{bridge}) {
1694         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1695     } else {
1696         $netdev = "type=user,id=$netid,hostname=$vmname";
1697     }
1698
1699     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1700
1701     return $netdev;
1702 }
1703
1704
1705 sub print_cpu_device {
1706     my ($conf, $id) = @_;
1707
1708     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
1709     my $cpu = $nokvm ? "qemu64" : "kvm64";
1710     if (my $cputype = $conf->{cpu}) {
1711         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1712             or die "Cannot parse cpu description: $cputype\n";
1713         $cpu = $cpuconf->{cputype};
1714     }
1715
1716     my $cores = $conf->{cores} || 1;
1717
1718     my $current_core = ($id - 1) % $cores;
1719     my $current_socket = int(($id - 1 - $current_core)/$cores);
1720
1721     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1722 }
1723
1724 sub drive_is_cdrom {
1725     my ($drive) = @_;
1726
1727     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1728
1729 }
1730
1731 sub parse_number_sets {
1732     my ($set) = @_;
1733     my $res = [];
1734     foreach my $part (split(/;/, $set)) {
1735         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1736             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1737             push @$res, [ $1, $2 ];
1738         } else {
1739             die "invalid range: $part\n";
1740         }
1741     }
1742     return $res;
1743 }
1744
1745 sub parse_numa {
1746     my ($data) = @_;
1747
1748     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1749     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1750     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1751     return $res;
1752 }
1753
1754 sub parse_hostpci {
1755     my ($value) = @_;
1756
1757     return undef if !$value;
1758
1759     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1760
1761     my @idlist = split(/;/, $res->{host});
1762     delete $res->{host};
1763     foreach my $id (@idlist) {
1764         if ($id =~ /^$PCIRE$/) {
1765             if (defined($2)) {
1766                 push @{$res->{pciid}}, { id => $1, function => $2 };
1767             } else {
1768                 my $pcidevices = lspci($1);
1769                 $res->{pciid} = $pcidevices->{$1};
1770             }
1771         } else {
1772             # should have been caught by parse_property_string already
1773             die "failed to parse PCI id: $id\n";
1774         }
1775     }
1776     return $res;
1777 }
1778
1779 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1780 sub parse_net {
1781     my ($data) = @_;
1782
1783     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1784     if ($@) {
1785         warn $@;
1786         return undef;
1787     }
1788     if (!defined($res->{macaddr})) {
1789         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1790         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1791     }
1792     return $res;
1793 }
1794
1795 sub print_net {
1796     my $net = shift;
1797
1798     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1799 }
1800
1801 sub add_random_macs {
1802     my ($settings) = @_;
1803
1804     foreach my $opt (keys %$settings) {
1805         next if $opt !~ m/^net(\d+)$/;
1806         my $net = parse_net($settings->{$opt});
1807         next if !$net;
1808         $settings->{$opt} = print_net($net);
1809     }
1810 }
1811
1812 sub vm_is_volid_owner {
1813     my ($storecfg, $vmid, $volid) = @_;
1814
1815     if ($volid !~  m|^/|) {
1816         my ($path, $owner);
1817         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1818         if ($owner && ($owner == $vmid)) {
1819             return 1;
1820         }
1821     }
1822
1823     return undef;
1824 }
1825
1826 sub split_flagged_list {
1827     my $text = shift || '';
1828     $text =~ s/[,;]/ /g;
1829     $text =~ s/^\s+//;
1830     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1831 }
1832
1833 sub join_flagged_list {
1834     my ($how, $lst) = @_;
1835     join $how, map { $lst->{$_} . $_ } keys %$lst;
1836 }
1837
1838 sub vmconfig_delete_pending_option {
1839     my ($conf, $key, $force) = @_;
1840
1841     delete $conf->{pending}->{$key};
1842     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1843     $pending_delete_hash->{$key} = $force ? '!' : '';
1844     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1845 }
1846
1847 sub vmconfig_undelete_pending_option {
1848     my ($conf, $key) = @_;
1849
1850     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1851     delete $pending_delete_hash->{$key};
1852
1853     if (%$pending_delete_hash) {
1854         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1855     } else {
1856         delete $conf->{pending}->{delete};
1857     }
1858 }
1859
1860 sub vmconfig_register_unused_drive {
1861     my ($storecfg, $vmid, $conf, $drive) = @_;
1862
1863     if (!drive_is_cdrom($drive)) {
1864         my $volid = $drive->{file};
1865         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1866             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1867         }
1868     }
1869 }
1870
1871 sub vmconfig_cleanup_pending {
1872     my ($conf) = @_;
1873
1874     # remove pending changes when nothing changed
1875     my $changes;
1876     foreach my $opt (keys %{$conf->{pending}}) {
1877         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1878             $changes = 1;
1879             delete $conf->{pending}->{$opt};
1880         }
1881     }
1882
1883     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1884     my $pending_delete_hash = {};
1885     while (my ($opt, $force) = each %$current_delete_hash) {
1886         if (defined($conf->{$opt})) {
1887             $pending_delete_hash->{$opt} = $force;
1888         } else {
1889             $changes = 1;
1890         }
1891     }
1892
1893     if (%$pending_delete_hash) {
1894         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1895     } else {
1896         delete $conf->{pending}->{delete};
1897     }
1898
1899     return $changes;
1900 }
1901
1902 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1903 my $smbios1_fmt = {
1904     uuid => {
1905         type => 'string',
1906         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1907         format_description => 'UUID',
1908         description => "Set SMBIOS1 UUID.",
1909         optional => 1,
1910     },
1911     version => {
1912         type => 'string',
1913         pattern => '\S+',
1914         format_description => 'string',
1915         description => "Set SMBIOS1 version.",
1916         optional => 1,
1917     },
1918     serial => {
1919         type => 'string',
1920         pattern => '\S+',
1921         format_description => 'string',
1922         description => "Set SMBIOS1 serial number.",
1923         optional => 1,
1924     },
1925     manufacturer => {
1926         type => 'string',
1927         pattern => '\S+',
1928         format_description => 'string',
1929         description => "Set SMBIOS1 manufacturer.",
1930         optional => 1,
1931     },
1932     product => {
1933         type => 'string',
1934         pattern => '\S+',
1935         format_description => 'string',
1936         description => "Set SMBIOS1 product ID.",
1937         optional => 1,
1938     },
1939     sku => {
1940         type => 'string',
1941         pattern => '\S+',
1942         format_description => 'string',
1943         description => "Set SMBIOS1 SKU string.",
1944         optional => 1,
1945     },
1946     family => {
1947         type => 'string',
1948         pattern => '\S+',
1949         format_description => 'string',
1950         description => "Set SMBIOS1 family string.",
1951         optional => 1,
1952     },
1953 };
1954
1955 sub parse_smbios1 {
1956     my ($data) = @_;
1957
1958     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1959     warn $@ if $@;
1960     return $res;
1961 }
1962
1963 sub print_smbios1 {
1964     my ($smbios1) = @_;
1965     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
1966 }
1967
1968 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
1969
1970 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
1971 sub verify_bootdisk {
1972     my ($value, $noerr) = @_;
1973
1974     return $value if is_valid_drivename($value);
1975
1976     return undef if $noerr;
1977
1978     die "invalid boot disk '$value'\n";
1979 }
1980
1981 sub parse_watchdog {
1982     my ($value) = @_;
1983
1984     return undef if !$value;
1985
1986     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
1987     warn $@ if $@;
1988     return $res;
1989 }
1990
1991 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
1992 sub verify_usb_device {
1993     my ($value, $noerr) = @_;
1994
1995     return $value if parse_usb_device($value);
1996
1997     return undef if $noerr;
1998
1999     die "unable to parse usb device\n";
2000 }
2001
2002 # add JSON properties for create and set function
2003 sub json_config_properties {
2004     my $prop = shift;
2005
2006     foreach my $opt (keys %$confdesc) {
2007         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2008         $prop->{$opt} = $confdesc->{$opt};
2009     }
2010
2011     return $prop;
2012 }
2013
2014 sub check_type {
2015     my ($key, $value) = @_;
2016
2017     die "unknown setting '$key'\n" if !$confdesc->{$key};
2018
2019     my $type = $confdesc->{$key}->{type};
2020
2021     if (!defined($value)) {
2022         die "got undefined value\n";
2023     }
2024
2025     if ($value =~ m/[\n\r]/) {
2026         die "property contains a line feed\n";
2027     }
2028
2029     if ($type eq 'boolean') {
2030         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2031         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2032         die "type check ('boolean') failed - got '$value'\n";
2033     } elsif ($type eq 'integer') {
2034         return int($1) if $value =~ m/^(\d+)$/;
2035         die "type check ('integer') failed - got '$value'\n";
2036     } elsif ($type eq 'number') {
2037         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2038         die "type check ('number') failed - got '$value'\n";
2039     } elsif ($type eq 'string') {
2040         if (my $fmt = $confdesc->{$key}->{format}) {
2041             PVE::JSONSchema::check_format($fmt, $value);
2042             return $value;
2043         }
2044         $value =~ s/^\"(.*)\"$/$1/;
2045         return $value;
2046     } else {
2047         die "internal error"
2048     }
2049 }
2050
2051 sub check_iommu_support{
2052     #fixme : need to check IOMMU support
2053     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2054
2055     my $iommu=1;
2056     return $iommu;
2057
2058 }
2059
2060 sub touch_config {
2061     my ($vmid) = @_;
2062
2063     my $conf = PVE::QemuConfig->config_file($vmid);
2064     utime undef, undef, $conf;
2065 }
2066
2067 sub destroy_vm {
2068     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2069
2070     my $conffile = PVE::QemuConfig->config_file($vmid);
2071
2072     my $conf = PVE::QemuConfig->load_config($vmid);
2073
2074     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2075
2076     # only remove disks owned by this VM
2077     foreach_drive($conf, sub {
2078         my ($ds, $drive) = @_;
2079
2080         return if drive_is_cdrom($drive);
2081
2082         my $volid = $drive->{file};
2083
2084         return if !$volid || $volid =~ m|^/|;
2085
2086         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2087         return if !$path || !$owner || ($owner != $vmid);
2088
2089         eval {
2090             PVE::Storage::vdisk_free($storecfg, $volid);
2091         };
2092         warn "Could not remove disk '$volid', check manually: $@" if $@;
2093
2094     });
2095
2096     if ($keep_empty_config) {
2097         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2098     } else {
2099         unlink $conffile;
2100     }
2101
2102     # also remove unused disk
2103     eval {
2104         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2105
2106         eval {
2107             PVE::Storage::foreach_volid($dl, sub {
2108                 my ($volid, $sid, $volname, $d) = @_;
2109                 PVE::Storage::vdisk_free($storecfg, $volid);
2110             });
2111         };
2112         warn $@ if $@;
2113
2114     };
2115     warn $@ if $@;
2116 }
2117
2118 sub parse_vm_config {
2119     my ($filename, $raw) = @_;
2120
2121     return undef if !defined($raw);
2122
2123     my $res = {
2124         digest => Digest::SHA::sha1_hex($raw),
2125         snapshots => {},
2126         pending => {},
2127     };
2128
2129     $filename =~ m|/qemu-server/(\d+)\.conf$|
2130         || die "got strange filename '$filename'";
2131
2132     my $vmid = $1;
2133
2134     my $conf = $res;
2135     my $descr;
2136     my $section = '';
2137
2138     my @lines = split(/\n/, $raw);
2139     foreach my $line (@lines) {
2140         next if $line =~ m/^\s*$/;
2141
2142         if ($line =~ m/^\[PENDING\]\s*$/i) {
2143             $section = 'pending';
2144             if (defined($descr)) {
2145                 $descr =~ s/\s+$//;
2146                 $conf->{description} = $descr;
2147             }
2148             $descr = undef;
2149             $conf = $res->{$section} = {};
2150             next;
2151
2152         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2153             $section = $1;
2154             if (defined($descr)) {
2155                 $descr =~ s/\s+$//;
2156                 $conf->{description} = $descr;
2157             }
2158             $descr = undef;
2159             $conf = $res->{snapshots}->{$section} = {};
2160             next;
2161         }
2162
2163         if ($line =~ m/^\#(.*)\s*$/) {
2164             $descr = '' if !defined($descr);
2165             $descr .= PVE::Tools::decode_text($1) . "\n";
2166             next;
2167         }
2168
2169         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2170             $descr = '' if !defined($descr);
2171             $descr .= PVE::Tools::decode_text($2);
2172         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2173             $conf->{snapstate} = $1;
2174         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2175             my $key = $1;
2176             my $value = $2;
2177             $conf->{$key} = $value;
2178         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2179             my $value = $1;
2180             if ($section eq 'pending') {
2181                 $conf->{delete} = $value; # we parse this later
2182             } else {
2183                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2184             }
2185         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2186             my $key = $1;
2187             my $value = $2;
2188             eval { $value = check_type($key, $value); };
2189             if ($@) {
2190                 warn "vm $vmid - unable to parse value of '$key' - $@";
2191             } else {
2192                 $key = 'ide2' if $key eq 'cdrom';
2193                 my $fmt = $confdesc->{$key}->{format};
2194                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2195                     my $v = parse_drive($key, $value);
2196                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2197                         $v->{file} = $volid;
2198                         $value = print_drive($vmid, $v);
2199                     } else {
2200                         warn "vm $vmid - unable to parse value of '$key'\n";
2201                         next;
2202                     }
2203                 }
2204
2205                 $conf->{$key} = $value;
2206             }
2207         }
2208     }
2209
2210     if (defined($descr)) {
2211         $descr =~ s/\s+$//;
2212         $conf->{description} = $descr;
2213     }
2214     delete $res->{snapstate}; # just to be sure
2215
2216     return $res;
2217 }
2218
2219 sub write_vm_config {
2220     my ($filename, $conf) = @_;
2221
2222     delete $conf->{snapstate}; # just to be sure
2223
2224     if ($conf->{cdrom}) {
2225         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2226         $conf->{ide2} = $conf->{cdrom};
2227         delete $conf->{cdrom};
2228     }
2229
2230     # we do not use 'smp' any longer
2231     if ($conf->{sockets}) {
2232         delete $conf->{smp};
2233     } elsif ($conf->{smp}) {
2234         $conf->{sockets} = $conf->{smp};
2235         delete $conf->{cores};
2236         delete $conf->{smp};
2237     }
2238
2239     my $used_volids = {};
2240
2241     my $cleanup_config = sub {
2242         my ($cref, $pending, $snapname) = @_;
2243
2244         foreach my $key (keys %$cref) {
2245             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2246                 $key eq 'snapstate' || $key eq 'pending';
2247             my $value = $cref->{$key};
2248             if ($key eq 'delete') {
2249                 die "propertry 'delete' is only allowed in [PENDING]\n"
2250                     if !$pending;
2251                 # fixme: check syntax?
2252                 next;
2253             }
2254             eval { $value = check_type($key, $value); };
2255             die "unable to parse value of '$key' - $@" if $@;
2256
2257             $cref->{$key} = $value;
2258
2259             if (!$snapname && is_valid_drivename($key)) {
2260                 my $drive = parse_drive($key, $value);
2261                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2262             }
2263         }
2264     };
2265
2266     &$cleanup_config($conf);
2267
2268     &$cleanup_config($conf->{pending}, 1);
2269
2270     foreach my $snapname (keys %{$conf->{snapshots}}) {
2271         die "internal error" if $snapname eq 'pending';
2272         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2273     }
2274
2275     # remove 'unusedX' settings if we re-add a volume
2276     foreach my $key (keys %$conf) {
2277         my $value = $conf->{$key};
2278         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2279             delete $conf->{$key};
2280         }
2281     }
2282
2283     my $generate_raw_config = sub {
2284         my ($conf, $pending) = @_;
2285
2286         my $raw = '';
2287
2288         # add description as comment to top of file
2289         if (defined(my $descr = $conf->{description})) {
2290             if ($descr) {
2291                 foreach my $cl (split(/\n/, $descr)) {
2292                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2293                 }
2294             } else {
2295                 $raw .= "#\n" if $pending;
2296             }
2297         }
2298
2299         foreach my $key (sort keys %$conf) {
2300             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2301             $raw .= "$key: $conf->{$key}\n";
2302         }
2303         return $raw;
2304     };
2305
2306     my $raw = &$generate_raw_config($conf);
2307
2308     if (scalar(keys %{$conf->{pending}})){
2309         $raw .= "\n[PENDING]\n";
2310         $raw .= &$generate_raw_config($conf->{pending}, 1);
2311     }
2312
2313     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2314         $raw .= "\n[$snapname]\n";
2315         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2316     }
2317
2318     return $raw;
2319 }
2320
2321 sub load_defaults {
2322
2323     my $res = {};
2324
2325     # we use static defaults from our JSON schema configuration
2326     foreach my $key (keys %$confdesc) {
2327         if (defined(my $default = $confdesc->{$key}->{default})) {
2328             $res->{$key} = $default;
2329         }
2330     }
2331
2332     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2333     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2334
2335     return $res;
2336 }
2337
2338 sub config_list {
2339     my $vmlist = PVE::Cluster::get_vmlist();
2340     my $res = {};
2341     return $res if !$vmlist || !$vmlist->{ids};
2342     my $ids = $vmlist->{ids};
2343
2344     foreach my $vmid (keys %$ids) {
2345         my $d = $ids->{$vmid};
2346         next if !$d->{node} || $d->{node} ne $nodename;
2347         next if !$d->{type} || $d->{type} ne 'qemu';
2348         $res->{$vmid}->{exists} = 1;
2349     }
2350     return $res;
2351 }
2352
2353 # test if VM uses local resources (to prevent migration)
2354 sub check_local_resources {
2355     my ($conf, $noerr) = @_;
2356
2357     my $loc_res = 0;
2358
2359     $loc_res = 1 if $conf->{hostusb}; # old syntax
2360     $loc_res = 1 if $conf->{hostpci}; # old syntax
2361
2362     foreach my $k (keys %$conf) {
2363         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2364         # sockets are safe: they will recreated be on the target side post-migrate
2365         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2366         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2367     }
2368
2369     die "VM uses local resources\n" if $loc_res && !$noerr;
2370
2371     return $loc_res;
2372 }
2373
2374 # check if used storages are available on all nodes (use by migrate)
2375 sub check_storage_availability {
2376     my ($storecfg, $conf, $node) = @_;
2377
2378     foreach_drive($conf, sub {
2379         my ($ds, $drive) = @_;
2380
2381         my $volid = $drive->{file};
2382         return if !$volid;
2383
2384         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2385         return if !$sid;
2386
2387         # check if storage is available on both nodes
2388         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2389         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2390    });
2391 }
2392
2393 # list nodes where all VM images are available (used by has_feature API)
2394 sub shared_nodes {
2395     my ($conf, $storecfg) = @_;
2396
2397     my $nodelist = PVE::Cluster::get_nodelist();
2398     my $nodehash = { map { $_ => 1 } @$nodelist };
2399     my $nodename = PVE::INotify::nodename();
2400
2401     foreach_drive($conf, sub {
2402         my ($ds, $drive) = @_;
2403
2404         my $volid = $drive->{file};
2405         return if !$volid;
2406
2407         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2408         if ($storeid) {
2409             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2410             if ($scfg->{disable}) {
2411                 $nodehash = {};
2412             } elsif (my $avail = $scfg->{nodes}) {
2413                 foreach my $node (keys %$nodehash) {
2414                     delete $nodehash->{$node} if !$avail->{$node};
2415                 }
2416             } elsif (!$scfg->{shared}) {
2417                 foreach my $node (keys %$nodehash) {
2418                     delete $nodehash->{$node} if $node ne $nodename
2419                 }
2420             }
2421         }
2422     });
2423
2424     return $nodehash
2425 }
2426
2427 sub check_cmdline {
2428     my ($pidfile, $pid) = @_;
2429
2430     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2431     if (defined($fh)) {
2432         my $line = <$fh>;
2433         $fh->close;
2434         return undef if !$line;
2435         my @param = split(/\0/, $line);
2436
2437         my $cmd = $param[0];
2438         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2439
2440         for (my $i = 0; $i < scalar (@param); $i++) {
2441             my $p = $param[$i];
2442             next if !$p;
2443             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2444                 my $p = $param[$i+1];
2445                 return 1 if $p && ($p eq $pidfile);
2446                 return undef;
2447             }
2448         }
2449     }
2450     return undef;
2451 }
2452
2453 sub check_running {
2454     my ($vmid, $nocheck, $node) = @_;
2455
2456     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2457
2458     die "unable to find configuration file for VM $vmid - no such machine\n"
2459         if !$nocheck && ! -f $filename;
2460
2461     my $pidfile = pidfile_name($vmid);
2462
2463     if (my $fd = IO::File->new("<$pidfile")) {
2464         my $st = stat($fd);
2465         my $line = <$fd>;
2466         close($fd);
2467
2468         my $mtime = $st->mtime;
2469         if ($mtime > time()) {
2470             warn "file '$filename' modified in future\n";
2471         }
2472
2473         if ($line =~ m/^(\d+)$/) {
2474             my $pid = $1;
2475             if (check_cmdline($pidfile, $pid)) {
2476                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2477                     return $pid;
2478                 }
2479             }
2480         }
2481     }
2482
2483     return undef;
2484 }
2485
2486 sub vzlist {
2487
2488     my $vzlist = config_list();
2489
2490     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2491
2492     while (defined(my $de = $fd->read)) {
2493         next if $de !~ m/^(\d+)\.pid$/;
2494         my $vmid = $1;
2495         next if !defined($vzlist->{$vmid});
2496         if (my $pid = check_running($vmid)) {
2497             $vzlist->{$vmid}->{pid} = $pid;
2498         }
2499     }
2500
2501     return $vzlist;
2502 }
2503
2504 sub disksize {
2505     my ($storecfg, $conf) = @_;
2506
2507     my $bootdisk = $conf->{bootdisk};
2508     return undef if !$bootdisk;
2509     return undef if !is_valid_drivename($bootdisk);
2510
2511     return undef if !$conf->{$bootdisk};
2512
2513     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2514     return undef if !defined($drive);
2515
2516     return undef if drive_is_cdrom($drive);
2517
2518     my $volid = $drive->{file};
2519     return undef if !$volid;
2520
2521     return $drive->{size};
2522 }
2523
2524 my $last_proc_pid_stat;
2525
2526 # get VM status information
2527 # This must be fast and should not block ($full == false)
2528 # We only query KVM using QMP if $full == true (this can be slow)
2529 sub vmstatus {
2530     my ($opt_vmid, $full) = @_;
2531
2532     my $res = {};
2533
2534     my $storecfg = PVE::Storage::config();
2535
2536     my $list = vzlist();
2537     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2538
2539     my $cpucount = $cpuinfo->{cpus} || 1;
2540
2541     foreach my $vmid (keys %$list) {
2542         next if $opt_vmid && ($vmid ne $opt_vmid);
2543
2544         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2545         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2546
2547         my $d = {};
2548         $d->{pid} = $list->{$vmid}->{pid};
2549
2550         # fixme: better status?
2551         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2552
2553         my $size = disksize($storecfg, $conf);
2554         if (defined($size)) {
2555             $d->{disk} = 0; # no info available
2556             $d->{maxdisk} = $size;
2557         } else {
2558             $d->{disk} = 0;
2559             $d->{maxdisk} = 0;
2560         }
2561
2562         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2563         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2564         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2565
2566         $d->{name} = $conf->{name} || "VM $vmid";
2567         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2568
2569         if ($conf->{balloon}) {
2570             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2571             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2572         }
2573
2574         $d->{uptime} = 0;
2575         $d->{cpu} = 0;
2576         $d->{mem} = 0;
2577
2578         $d->{netout} = 0;
2579         $d->{netin} = 0;
2580
2581         $d->{diskread} = 0;
2582         $d->{diskwrite} = 0;
2583
2584         $d->{template} = PVE::QemuConfig->is_template($conf);
2585
2586         $res->{$vmid} = $d;
2587     }
2588
2589     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2590     foreach my $dev (keys %$netdev) {
2591         next if $dev !~ m/^tap([1-9]\d*)i/;
2592         my $vmid = $1;
2593         my $d = $res->{$vmid};
2594         next if !$d;
2595
2596         $d->{netout} += $netdev->{$dev}->{receive};
2597         $d->{netin} += $netdev->{$dev}->{transmit};
2598
2599         if ($full) {
2600             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2601             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2602         }
2603
2604     }
2605
2606     my $ctime = gettimeofday;
2607
2608     foreach my $vmid (keys %$list) {
2609
2610         my $d = $res->{$vmid};
2611         my $pid = $d->{pid};
2612         next if !$pid;
2613
2614         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2615         next if !$pstat; # not running
2616
2617         my $used = $pstat->{utime} + $pstat->{stime};
2618
2619         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2620
2621         if ($pstat->{vsize}) {
2622             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2623         }
2624
2625         my $old = $last_proc_pid_stat->{$pid};
2626         if (!$old) {
2627             $last_proc_pid_stat->{$pid} = {
2628                 time => $ctime,
2629                 used => $used,
2630                 cpu => 0,
2631             };
2632             next;
2633         }
2634
2635         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2636
2637         if ($dtime > 1000) {
2638             my $dutime = $used -  $old->{used};
2639
2640             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2641             $last_proc_pid_stat->{$pid} = {
2642                 time => $ctime,
2643                 used => $used,
2644                 cpu => $d->{cpu},
2645             };
2646         } else {
2647             $d->{cpu} = $old->{cpu};
2648         }
2649     }
2650
2651     return $res if !$full;
2652
2653     my $qmpclient = PVE::QMPClient->new();
2654
2655     my $ballooncb = sub {
2656         my ($vmid, $resp) = @_;
2657
2658         my $info = $resp->{'return'};
2659         return if !$info->{max_mem};
2660
2661         my $d = $res->{$vmid};
2662
2663         # use memory assigned to VM
2664         $d->{maxmem} = $info->{max_mem};
2665         $d->{balloon} = $info->{actual};
2666
2667         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2668             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2669             $d->{freemem} = $info->{free_mem};
2670         }
2671
2672         $d->{ballooninfo} = $info;
2673     };
2674
2675     my $blockstatscb = sub {
2676         my ($vmid, $resp) = @_;
2677         my $data = $resp->{'return'} || [];
2678         my $totalrdbytes = 0;
2679         my $totalwrbytes = 0;
2680
2681         for my $blockstat (@$data) {
2682             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2683             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2684
2685             $blockstat->{device} =~ s/drive-//;
2686             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2687         }
2688         $res->{$vmid}->{diskread} = $totalrdbytes;
2689         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2690     };
2691
2692     my $statuscb = sub {
2693         my ($vmid, $resp) = @_;
2694
2695         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2696         # this fails if ballon driver is not loaded, so this must be
2697         # the last commnand (following command are aborted if this fails).
2698         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2699
2700         my $status = 'unknown';
2701         if (!defined($status = $resp->{'return'}->{status})) {
2702             warn "unable to get VM status\n";
2703             return;
2704         }
2705
2706         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2707     };
2708
2709     foreach my $vmid (keys %$list) {
2710         next if $opt_vmid && ($vmid ne $opt_vmid);
2711         next if !$res->{$vmid}->{pid}; # not running
2712         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2713     }
2714
2715     $qmpclient->queue_execute(undef, 2);
2716
2717     foreach my $vmid (keys %$list) {
2718         next if $opt_vmid && ($vmid ne $opt_vmid);
2719         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2720     }
2721
2722     return $res;
2723 }
2724
2725 sub foreach_drive {
2726     my ($conf, $func, @param) = @_;
2727
2728     foreach my $ds (valid_drive_names()) {
2729         next if !defined($conf->{$ds});
2730
2731         my $drive = parse_drive($ds, $conf->{$ds});
2732         next if !$drive;
2733
2734         &$func($ds, $drive, @param);
2735     }
2736 }
2737
2738 sub foreach_volid {
2739     my ($conf, $func, @param) = @_;
2740
2741     my $volhash = {};
2742
2743     my $test_volid = sub {
2744         my ($volid, $is_cdrom) = @_;
2745
2746         return if !$volid;
2747
2748         $volhash->{$volid} = $is_cdrom || 0;
2749     };
2750
2751     foreach_drive($conf, sub {
2752         my ($ds, $drive) = @_;
2753         &$test_volid($drive->{file}, drive_is_cdrom($drive));
2754     });
2755
2756     foreach my $snapname (keys %{$conf->{snapshots}}) {
2757         my $snap = $conf->{snapshots}->{$snapname};
2758         &$test_volid($snap->{vmstate}, 0);
2759         foreach_drive($snap, sub {
2760             my ($ds, $drive) = @_;
2761             &$test_volid($drive->{file}, drive_is_cdrom($drive));
2762         });
2763     }
2764
2765     foreach my $volid (keys %$volhash) {
2766         &$func($volid, $volhash->{$volid}, @param);
2767     }
2768 }
2769
2770 sub vga_conf_has_spice {
2771     my ($vga) = @_;
2772
2773     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2774
2775     return $1 || 1;
2776 }
2777
2778 sub config_to_command {
2779     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2780
2781     my $cmd = [];
2782     my $globalFlags = [];
2783     my $machineFlags = [];
2784     my $rtcFlags = [];
2785     my $cpuFlags = [];
2786     my $devices = [];
2787     my $pciaddr = '';
2788     my $bridges = {};
2789     my $kvmver = kvm_user_version();
2790     my $vernum = 0; # unknown
2791     my $ostype = $conf->{ostype};
2792     my $winversion = windows_version($ostype);
2793
2794     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2795         $vernum = $1*1000000+$2*1000;
2796     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2797         $vernum = $1*1000000+$2*1000+$3;
2798     }
2799
2800     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2801
2802     my $have_ovz = -f '/proc/vz/vestat';
2803
2804     my $q35 = machine_type_is_q35($conf);
2805     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2806     my $machine_type = $forcemachine || $conf->{machine};
2807     my $use_old_bios_files = undef;
2808     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2809
2810     my $cpuunits = defined($conf->{cpuunits}) ?
2811             $conf->{cpuunits} : $defaults->{cpuunits};
2812
2813     push @$cmd, '/usr/bin/kvm';
2814
2815     push @$cmd, '-id', $vmid;
2816
2817     my $use_virtio = 0;
2818
2819     my $qmpsocket = qmp_socket($vmid);
2820     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2821     push @$cmd, '-mon', "chardev=qmp,mode=control";
2822
2823
2824     push @$cmd, '-pidfile' , pidfile_name($vmid);
2825
2826     push @$cmd, '-daemonize';
2827
2828     if ($conf->{smbios1}) {
2829         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2830     }
2831
2832     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2833         my $ovmfbase;
2834
2835         # prefer the OVMF_CODE variant
2836         if (-f $OVMF_CODE) {
2837             $ovmfbase = $OVMF_CODE;
2838         } elsif (-f $OVMF_IMG) {
2839             $ovmfbase = $OVMF_IMG;
2840         }
2841
2842         die "no uefi base img found\n" if !$ovmfbase;
2843         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$ovmfbase";
2844
2845         if (defined($conf->{efidisk0}) && ($ovmfbase eq $OVMF_CODE)) {
2846             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $conf->{efidisk0});
2847             my $format = $d->{format} // 'raw';
2848             my $path;
2849             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2850             if ($storeid) {
2851                 $path = PVE::Storage::path($storecfg, $d->{file});
2852                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2853                 $format = qemu_img_format($scfg, $volname);
2854             } else {
2855                 $path = $d->{file};
2856                 $format = "raw";
2857             }
2858             push @$cmd, '-drive', "if=pflash,unit=1,id=drive-efidisk0,format=$format,file=$path";
2859         } elsif ($ovmfbase eq $OVMF_CODE) {
2860             warn "using uefi without permanent efivars disk\n";
2861             my $ovmfvar_dst = "/tmp/$vmid-ovmf.fd";
2862             PVE::Tools::file_copy($OVMF_VARS, $ovmfvar_dst, 256*1024);
2863             push @$cmd, '-drive', "if=pflash,unit=1,format=raw,file=$ovmfvar_dst";
2864         } else {
2865             # if the base img is not OVMF_CODE, we do not have to bother
2866             # to create/use a vars image, since it will not be used anyway
2867             # this can only happen if someone manually deletes the OVMF_CODE image
2868             # or has an old pve-qemu-kvm version installed.
2869             # both should not happen, but we ignore it here
2870         }
2871     }
2872
2873
2874     # add usb controllers
2875     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2876     push @$devices, @usbcontrollers if @usbcontrollers;
2877     my $vga = $conf->{vga};
2878
2879     my $qxlnum = vga_conf_has_spice($vga);
2880     $vga = 'qxl' if $qxlnum;
2881
2882     if (!$vga) {
2883         $vga = $winversion >= 6 ? 'std' : 'cirrus';
2884     }
2885
2886     # enable absolute mouse coordinates (needed by vnc)
2887     my $tablet;
2888     if (defined($conf->{tablet})) {
2889         $tablet = $conf->{tablet};
2890     } else {
2891         $tablet = $defaults->{tablet};
2892         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2893         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2894     }
2895
2896     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2897
2898     my $kvm_off = 0;
2899     my $gpu_passthrough;
2900
2901     # host pci devices
2902     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2903         my $d = parse_hostpci($conf->{"hostpci$i"});
2904         next if !$d;
2905
2906         my $pcie = $d->{pcie};
2907         if($pcie){
2908             die "q35 machine model is not enabled" if !$q35;
2909             $pciaddr = print_pcie_addr("hostpci$i");
2910         }else{
2911             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2912         }
2913
2914         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2915         my $romfile = $d->{romfile};
2916
2917         my $xvga = '';
2918         if ($d->{'x-vga'}) {
2919             $xvga = ',x-vga=on';
2920             $kvm_off = 1;
2921             $vga = 'none';
2922             $gpu_passthrough = 1;
2923
2924             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2925                 $xvga = "";
2926             }
2927         }
2928         my $pcidevices = $d->{pciid};
2929         my $multifunction = 1 if @$pcidevices > 1;
2930
2931         my $j=0;
2932         foreach my $pcidevice (@$pcidevices) {
2933
2934             my $id = "hostpci$i";
2935             $id .= ".$j" if $multifunction;
2936             my $addr = $pciaddr;
2937             $addr .= ".$j" if $multifunction;
2938             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2939
2940             if($j == 0){
2941                 $devicestr .= "$rombar$xvga";
2942                 $devicestr .= ",multifunction=on" if $multifunction;
2943                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
2944             }
2945
2946             push @$devices, '-device', $devicestr;
2947             $j++;
2948         }
2949     }
2950
2951     # usb devices
2952     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
2953     push @$devices, @usbdevices if @usbdevices;
2954     # serial devices
2955     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2956         if (my $path = $conf->{"serial$i"}) {
2957             if ($path eq 'socket') {
2958                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
2959                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
2960                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2961             } else {
2962                 die "no such serial device\n" if ! -c $path;
2963                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
2964                 push @$devices, '-device', "isa-serial,chardev=serial$i";
2965             }
2966         }
2967     }
2968
2969     # parallel devices
2970     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
2971         if (my $path = $conf->{"parallel$i"}) {
2972             die "no such parallel device\n" if ! -c $path;
2973             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
2974             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
2975             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
2976         }
2977     }
2978
2979     my $vmname = $conf->{name} || "vm$vmid";
2980
2981     push @$cmd, '-name', $vmname;
2982
2983     my $sockets = 1;
2984     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
2985     $sockets = $conf->{sockets} if  $conf->{sockets};
2986
2987     my $cores = $conf->{cores} || 1;
2988
2989     my $maxcpus = $sockets * $cores;
2990
2991     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
2992
2993     my $allowed_vcpus = $cpuinfo->{cpus};
2994
2995     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
2996         if ($allowed_vcpus < $maxcpus);
2997
2998     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
2999
3000         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3001         for (my $i = 2; $i <= $vcpus; $i++)  {
3002             my $cpustr = print_cpu_device($conf,$i);
3003             push @$cmd, '-device', $cpustr;
3004         }
3005
3006     } else {
3007
3008         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3009     }
3010     push @$cmd, '-nodefaults';
3011
3012     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3013
3014     my $bootindex_hash = {};
3015     my $i = 1;
3016     foreach my $o (split(//, $bootorder)) {
3017         $bootindex_hash->{$o} = $i*100;
3018         $i++;
3019     }
3020
3021     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3022
3023     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3024
3025     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3026
3027     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3028
3029     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3030         my $socket = vnc_socket($vmid);
3031         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3032     } else {
3033         push @$cmd, '-nographic';
3034     }
3035
3036     # time drift fix
3037     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3038
3039     my $nokvm = defined($conf->{kvm}) && $conf->{kvm} == 0 ? 1 : 0;
3040     my $useLocaltime = $conf->{localtime};
3041
3042     if ($winversion >= 5) { # windows
3043         $useLocaltime = 1 if !defined($conf->{localtime});
3044
3045         # use time drift fix when acpi is enabled
3046         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3047             $tdf = 1 if !defined($conf->{tdf});
3048         }
3049     }
3050
3051     if ($winversion >= 6) {
3052         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3053         push @$cmd, '-no-hpet';
3054     }
3055
3056     push @$rtcFlags, 'driftfix=slew' if $tdf;
3057
3058     if ($nokvm) {
3059         push @$machineFlags, 'accel=tcg';
3060     } else {
3061         die "No accelerator found!\n" if !$cpuinfo->{hvm};
3062     }
3063
3064     if ($machine_type) {
3065         push @$machineFlags, "type=${machine_type}";
3066     }
3067
3068     if ($conf->{startdate}) {
3069         push @$rtcFlags, "base=$conf->{startdate}";
3070     } elsif ($useLocaltime) {
3071         push @$rtcFlags, 'base=localtime';
3072     }
3073
3074     my $cpu = $nokvm ? "qemu64" : "kvm64";
3075     if (my $cputype = $conf->{cpu}) {
3076         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3077             or die "Cannot parse cpu description: $cputype\n";
3078         $cpu = $cpuconf->{cputype};
3079         $kvm_off = 1 if $cpuconf->{hidden};
3080     }
3081
3082     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3083
3084     push @$cpuFlags , '-x2apic'
3085         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3086
3087     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3088
3089     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3090
3091     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3092
3093         push @$cpuFlags , '+kvm_pv_unhalt' if !$nokvm;
3094         push @$cpuFlags , '+kvm_pv_eoi' if !$nokvm;
3095     }
3096
3097     add_hyperv_enlighments($cpuFlags, $winversion, $machine_type, $kvmver, $nokvm, $conf->{bios}, $gpu_passthrough);
3098
3099     push @$cpuFlags, 'enforce' if $cpu ne 'host' && !$nokvm;
3100
3101     push @$cpuFlags, 'kvm=off' if $kvm_off;
3102
3103     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3104         die "internal error"; # should not happen
3105
3106     push @$cpuFlags, "vendor=${cpu_vendor}"
3107         if $cpu_vendor ne 'default';
3108
3109     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3110
3111     push @$cmd, '-cpu', $cpu;
3112
3113     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3114
3115     push @$cmd, '-S' if $conf->{freeze};
3116
3117     # set keyboard layout
3118     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3119     push @$cmd, '-k', $kb if $kb;
3120
3121     # enable sound
3122     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3123     #push @$cmd, '-soundhw', 'es1370';
3124     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3125
3126     if($conf->{agent}) {
3127         my $qgasocket = qmp_socket($vmid, 1);
3128         my $pciaddr = print_pci_addr("qga0", $bridges);
3129         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3130         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3131         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3132     }
3133
3134     my $spice_port;
3135
3136     if ($qxlnum) {
3137         if ($qxlnum > 1) {
3138             if ($winversion){
3139                 for(my $i = 1; $i < $qxlnum; $i++){
3140                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3141                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3142                 }
3143             } else {
3144                 # assume other OS works like Linux
3145                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3146                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3147             }
3148         }
3149
3150         my $pciaddr = print_pci_addr("spice", $bridges);
3151
3152         my $nodename = PVE::INotify::nodename();
3153         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3154         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3155         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3156         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3157         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3158
3159         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3160
3161         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3162         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3163         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3164     }
3165
3166     # enable balloon by default, unless explicitly disabled
3167     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3168         $pciaddr = print_pci_addr("balloon0", $bridges);
3169         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3170     }
3171
3172     if ($conf->{watchdog}) {
3173         my $wdopts = parse_watchdog($conf->{watchdog});
3174         $pciaddr = print_pci_addr("watchdog", $bridges);
3175         my $watchdog = $wdopts->{model} || 'i6300esb';
3176         push @$devices, '-device', "$watchdog$pciaddr";
3177         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3178     }
3179
3180     my $vollist = [];
3181     my $scsicontroller = {};
3182     my $ahcicontroller = {};
3183     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3184
3185     # Add iscsi initiator name if available
3186     if (my $initiator = get_initiator_name()) {
3187         push @$devices, '-iscsi', "initiator-name=$initiator";
3188     }
3189
3190     foreach_drive($conf, sub {
3191         my ($ds, $drive) = @_;
3192
3193         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3194             push @$vollist, $drive->{file};
3195         }
3196
3197         $use_virtio = 1 if $ds =~ m/^virtio/;
3198
3199         if (drive_is_cdrom ($drive)) {
3200             if ($bootindex_hash->{d}) {
3201                 $drive->{bootindex} = $bootindex_hash->{d};
3202                 $bootindex_hash->{d} += 1;
3203             }
3204         } else {
3205             if ($bootindex_hash->{c}) {
3206                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3207                 $bootindex_hash->{c} += 1;
3208             }
3209         }
3210
3211         if($drive->{interface} eq 'virtio'){
3212            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3213         }
3214
3215         if ($drive->{interface} eq 'scsi') {
3216
3217             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3218
3219             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3220             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3221
3222             my $iothread = '';
3223             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3224                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3225                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3226             } elsif ($drive->{iothread}) {
3227                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3228             }
3229
3230             my $queues = '';
3231             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3232                 $queues = ",num_queues=$drive->{queues}";
3233             }
3234
3235             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3236             $scsicontroller->{$controller}=1;
3237         }
3238
3239         if ($drive->{interface} eq 'sata') {
3240            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3241            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3242            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3243            $ahcicontroller->{$controller}=1;
3244         }
3245
3246         if ($drive->{interface} eq 'efidisk') {
3247             # this will be added somewhere else
3248             return;
3249         }
3250
3251         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3252         push @$devices, '-drive',$drive_cmd;
3253         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3254     });
3255
3256     for (my $i = 0; $i < $MAX_NETS; $i++) {
3257          next if !$conf->{"net$i"};
3258          my $d = parse_net($conf->{"net$i"});
3259          next if !$d;
3260
3261          $use_virtio = 1 if $d->{model} eq 'virtio';
3262
3263          if ($bootindex_hash->{n}) {
3264             $d->{bootindex} = $bootindex_hash->{n};
3265             $bootindex_hash->{n} += 1;
3266          }
3267
3268          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3269          push @$devices, '-netdev', $netdevfull;
3270
3271          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3272          push @$devices, '-device', $netdevicefull;
3273     }
3274
3275     if (!$q35) {
3276         # add pci bridges
3277         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3278            $bridges->{1} = 1;
3279            $bridges->{2} = 1;
3280         }
3281
3282         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3283
3284         while (my ($k, $v) = each %$bridges) {
3285             $pciaddr = print_pci_addr("pci.$k");
3286             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3287         }
3288     }
3289
3290     # add custom args
3291     if ($conf->{args}) {
3292         my $aa = PVE::Tools::split_args($conf->{args});
3293         push @$cmd, @$aa;
3294     }
3295
3296     push @$cmd, @$devices;
3297     push @$cmd, '-rtc', join(',', @$rtcFlags)
3298         if scalar(@$rtcFlags);
3299     push @$cmd, '-machine', join(',', @$machineFlags)
3300         if scalar(@$machineFlags);
3301     push @$cmd, '-global', join(',', @$globalFlags)
3302         if scalar(@$globalFlags);
3303
3304     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3305 }
3306
3307 sub vnc_socket {
3308     my ($vmid) = @_;
3309     return "${var_run_tmpdir}/$vmid.vnc";
3310 }
3311
3312 sub spice_port {
3313     my ($vmid) = @_;
3314
3315     my $res = vm_mon_cmd($vmid, 'query-spice');
3316
3317     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3318 }
3319
3320 sub qmp_socket {
3321     my ($vmid, $qga) = @_;
3322     my $sockettype = $qga ? 'qga' : 'qmp';
3323     return "${var_run_tmpdir}/$vmid.$sockettype";
3324 }
3325
3326 sub pidfile_name {
3327     my ($vmid) = @_;
3328     return "${var_run_tmpdir}/$vmid.pid";
3329 }
3330
3331 sub vm_devices_list {
3332     my ($vmid) = @_;
3333
3334     my $res = vm_mon_cmd($vmid, 'query-pci');
3335     my $devices = {};
3336     foreach my $pcibus (@$res) {
3337         foreach my $device (@{$pcibus->{devices}}) {
3338             next if !$device->{'qdev_id'};
3339             if ($device->{'pci_bridge'}) {
3340                 $devices->{$device->{'qdev_id'}} = 1;
3341                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3342                     next if !$bridge_device->{'qdev_id'};
3343                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3344                     $devices->{$device->{'qdev_id'}}++;
3345                 }
3346             } else {
3347                 $devices->{$device->{'qdev_id'}} = 1;
3348             }
3349         }
3350     }
3351
3352     my $resblock = vm_mon_cmd($vmid, 'query-block');
3353     foreach my $block (@$resblock) {
3354         if($block->{device} =~ m/^drive-(\S+)/){
3355                 $devices->{$1} = 1;
3356         }
3357     }
3358
3359     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3360     foreach my $mice (@$resmice) {
3361         if ($mice->{name} eq 'QEMU HID Tablet') {
3362             $devices->{tablet} = 1;
3363             last;
3364         }
3365     }
3366
3367     # for usb devices there is no query-usb
3368     # but we can iterate over the entries in
3369     # qom-list path=/machine/peripheral
3370     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3371     foreach my $per (@$resperipheral) {
3372         if ($per->{name} =~ m/^usb\d+$/) {
3373             $devices->{$per->{name}} = 1;
3374         }
3375     }
3376
3377     return $devices;
3378 }
3379
3380 sub vm_deviceplug {
3381     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3382
3383     my $q35 = machine_type_is_q35($conf);
3384
3385     my $devices_list = vm_devices_list($vmid);
3386     return 1 if defined($devices_list->{$deviceid});
3387
3388     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3389
3390     if ($deviceid eq 'tablet') {
3391
3392         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3393
3394     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3395
3396         die "usb hotplug currently not reliable\n";
3397         # since we can't reliably hot unplug all added usb devices
3398         # and usb passthrough disables live migration
3399         # we disable usb hotplugging for now
3400         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3401
3402     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3403
3404         qemu_iothread_add($vmid, $deviceid, $device);
3405
3406         qemu_driveadd($storecfg, $vmid, $device);
3407         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3408
3409         qemu_deviceadd($vmid, $devicefull);
3410         eval { qemu_deviceaddverify($vmid, $deviceid); };
3411         if (my $err = $@) {
3412             eval { qemu_drivedel($vmid, $deviceid); };
3413             warn $@ if $@;
3414             die $err;
3415         }
3416
3417     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3418
3419
3420         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3421         my $pciaddr = print_pci_addr($deviceid);
3422         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3423
3424         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3425
3426         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3427             qemu_iothread_add($vmid, $deviceid, $device);
3428             $devicefull .= ",iothread=iothread-$deviceid";
3429         }
3430
3431         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3432             $devicefull .= ",num_queues=$device->{queues}";
3433         }
3434
3435         qemu_deviceadd($vmid, $devicefull);
3436         qemu_deviceaddverify($vmid, $deviceid);
3437
3438     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3439
3440         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3441         qemu_driveadd($storecfg, $vmid, $device);
3442
3443         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3444         eval { qemu_deviceadd($vmid, $devicefull); };
3445         if (my $err = $@) {
3446             eval { qemu_drivedel($vmid, $deviceid); };
3447             warn $@ if $@;
3448             die $err;
3449         }
3450
3451     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3452
3453         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3454
3455         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3456         my $use_old_bios_files = undef;
3457         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3458
3459         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3460         qemu_deviceadd($vmid, $netdevicefull);
3461         eval { qemu_deviceaddverify($vmid, $deviceid); };
3462         if (my $err = $@) {
3463             eval { qemu_netdevdel($vmid, $deviceid); };
3464             warn $@ if $@;
3465             die $err;
3466         }
3467
3468     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3469
3470         my $bridgeid = $2;
3471         my $pciaddr = print_pci_addr($deviceid);
3472         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3473
3474         qemu_deviceadd($vmid, $devicefull);
3475         qemu_deviceaddverify($vmid, $deviceid);
3476
3477     } else {
3478         die "can't hotplug device '$deviceid'\n";
3479     }
3480
3481     return 1;
3482 }
3483
3484 # fixme: this should raise exceptions on error!
3485 sub vm_deviceunplug {
3486     my ($vmid, $conf, $deviceid) = @_;
3487
3488     my $devices_list = vm_devices_list($vmid);
3489     return 1 if !defined($devices_list->{$deviceid});
3490
3491     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3492
3493     if ($deviceid eq 'tablet') {
3494
3495         qemu_devicedel($vmid, $deviceid);
3496
3497     } elsif ($deviceid =~ m/^usb\d+$/) {
3498
3499         die "usb hotplug currently not reliable\n";
3500         # when unplugging usb devices this way,
3501         # there may be remaining usb controllers/hubs
3502         # so we disable it for now
3503         qemu_devicedel($vmid, $deviceid);
3504         qemu_devicedelverify($vmid, $deviceid);
3505
3506     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3507
3508         qemu_devicedel($vmid, $deviceid);
3509         qemu_devicedelverify($vmid, $deviceid);
3510         qemu_drivedel($vmid, $deviceid);
3511         qemu_iothread_del($conf, $vmid, $deviceid);
3512
3513     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3514
3515         qemu_devicedel($vmid, $deviceid);
3516         qemu_devicedelverify($vmid, $deviceid);
3517         qemu_iothread_del($conf, $vmid, $deviceid);
3518
3519     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3520
3521         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3522         my $device = parse_drive($deviceid, $conf->{$deviceid});
3523         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3524
3525         qemu_devicedel($vmid, $deviceid);
3526         qemu_drivedel($vmid, $deviceid);
3527         qemu_deletescsihw($conf, $vmid, $deviceid);
3528
3529     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3530
3531         qemu_devicedel($vmid, $deviceid);
3532         qemu_devicedelverify($vmid, $deviceid);
3533         qemu_netdevdel($vmid, $deviceid);
3534
3535     } else {
3536         die "can't unplug device '$deviceid'\n";
3537     }
3538
3539     return 1;
3540 }
3541
3542 sub qemu_deviceadd {
3543     my ($vmid, $devicefull) = @_;
3544
3545     $devicefull = "driver=".$devicefull;
3546     my %options =  split(/[=,]/, $devicefull);
3547
3548     vm_mon_cmd($vmid, "device_add" , %options);
3549 }
3550
3551 sub qemu_devicedel {
3552     my ($vmid, $deviceid) = @_;
3553
3554     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3555 }
3556
3557 sub qemu_iothread_add {
3558     my($vmid, $deviceid, $device) = @_;
3559
3560     if ($device->{iothread}) {
3561         my $iothreads = vm_iothreads_list($vmid);
3562         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3563     }
3564 }
3565
3566 sub qemu_iothread_del {
3567     my($conf, $vmid, $deviceid) = @_;
3568
3569     my $device = parse_drive($deviceid, $conf->{$deviceid});
3570     if ($device->{iothread}) {
3571         my $iothreads = vm_iothreads_list($vmid);
3572         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3573     }
3574 }
3575
3576 sub qemu_objectadd {
3577     my($vmid, $objectid, $qomtype) = @_;
3578
3579     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3580
3581     return 1;
3582 }
3583
3584 sub qemu_objectdel {
3585     my($vmid, $objectid) = @_;
3586
3587     vm_mon_cmd($vmid, "object-del", id => $objectid);
3588
3589     return 1;
3590 }
3591
3592 sub qemu_driveadd {
3593     my ($storecfg, $vmid, $device) = @_;
3594
3595     my $drive = print_drive_full($storecfg, $vmid, $device);
3596     $drive =~ s/\\/\\\\/g;
3597     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3598
3599     # If the command succeeds qemu prints: "OK"
3600     return 1 if $ret =~ m/OK/s;
3601
3602     die "adding drive failed: $ret\n";
3603 }
3604
3605 sub qemu_drivedel {
3606     my($vmid, $deviceid) = @_;
3607
3608     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3609     $ret =~ s/^\s+//;
3610
3611     return 1 if $ret eq "";
3612
3613     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3614     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3615
3616     die "deleting drive $deviceid failed : $ret\n";
3617 }
3618
3619 sub qemu_deviceaddverify {
3620     my ($vmid, $deviceid) = @_;
3621
3622     for (my $i = 0; $i <= 5; $i++) {
3623          my $devices_list = vm_devices_list($vmid);
3624          return 1 if defined($devices_list->{$deviceid});
3625          sleep 1;
3626     }
3627
3628     die "error on hotplug device '$deviceid'\n";
3629 }
3630
3631
3632 sub qemu_devicedelverify {
3633     my ($vmid, $deviceid) = @_;
3634
3635     # need to verify that the device is correctly removed as device_del
3636     # is async and empty return is not reliable
3637
3638     for (my $i = 0; $i <= 5; $i++) {
3639          my $devices_list = vm_devices_list($vmid);
3640          return 1 if !defined($devices_list->{$deviceid});
3641          sleep 1;
3642     }
3643
3644     die "error on hot-unplugging device '$deviceid'\n";
3645 }
3646
3647 sub qemu_findorcreatescsihw {
3648     my ($storecfg, $conf, $vmid, $device) = @_;
3649
3650     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3651
3652     my $scsihwid="$controller_prefix$controller";
3653     my $devices_list = vm_devices_list($vmid);
3654
3655     if(!defined($devices_list->{$scsihwid})) {
3656         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3657     }
3658
3659     return 1;
3660 }
3661
3662 sub qemu_deletescsihw {
3663     my ($conf, $vmid, $opt) = @_;
3664
3665     my $device = parse_drive($opt, $conf->{$opt});
3666
3667     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3668         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3669         return 1;
3670     }
3671
3672     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3673
3674     my $devices_list = vm_devices_list($vmid);
3675     foreach my $opt (keys %{$devices_list}) {
3676         if (PVE::QemuServer::is_valid_drivename($opt)) {
3677             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3678             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3679                 return 1;
3680             }
3681         }
3682     }
3683
3684     my $scsihwid="scsihw$controller";
3685
3686     vm_deviceunplug($vmid, $conf, $scsihwid);
3687
3688     return 1;
3689 }
3690
3691 sub qemu_add_pci_bridge {
3692     my ($storecfg, $conf, $vmid, $device) = @_;
3693
3694     my $bridges = {};
3695
3696     my $bridgeid;
3697
3698     print_pci_addr($device, $bridges);
3699
3700     while (my ($k, $v) = each %$bridges) {
3701         $bridgeid = $k;
3702     }
3703     return 1 if !defined($bridgeid) || $bridgeid < 1;
3704
3705     my $bridge = "pci.$bridgeid";
3706     my $devices_list = vm_devices_list($vmid);
3707
3708     if (!defined($devices_list->{$bridge})) {
3709         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3710     }
3711
3712     return 1;
3713 }
3714
3715 sub qemu_set_link_status {
3716     my ($vmid, $device, $up) = @_;
3717
3718     vm_mon_cmd($vmid, "set_link", name => $device,
3719                up => $up ? JSON::true : JSON::false);
3720 }
3721
3722 sub qemu_netdevadd {
3723     my ($vmid, $conf, $device, $deviceid) = @_;
3724
3725     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3726     my %options =  split(/[=,]/, $netdev);
3727
3728     vm_mon_cmd($vmid, "netdev_add",  %options);
3729     return 1;
3730 }
3731
3732 sub qemu_netdevdel {
3733     my ($vmid, $deviceid) = @_;
3734
3735     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3736 }
3737
3738 sub qemu_usb_hotplug {
3739     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3740
3741     return if !$device;
3742
3743     # remove the old one first
3744     vm_deviceunplug($vmid, $conf, $deviceid);
3745
3746     # check if xhci controller is necessary and available
3747     if ($device->{usb3}) {
3748
3749         my $devicelist = vm_devices_list($vmid);
3750
3751         if (!$devicelist->{xhci}) {
3752             my $pciaddr = print_pci_addr("xhci");
3753             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3754         }
3755     }
3756     my $d = parse_usb_device($device->{host});
3757     $d->{usb3} = $device->{usb3};
3758
3759     # add the new one
3760     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3761 }
3762
3763 sub qemu_cpu_hotplug {
3764     my ($vmid, $conf, $vcpus) = @_;
3765
3766     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3767
3768     my $sockets = 1;
3769     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3770     $sockets = $conf->{sockets} if  $conf->{sockets};
3771     my $cores = $conf->{cores} || 1;
3772     my $maxcpus = $sockets * $cores;
3773
3774     $vcpus = $maxcpus if !$vcpus;
3775
3776     die "you can't add more vcpus than maxcpus\n"
3777         if $vcpus > $maxcpus;
3778
3779     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3780
3781     if ($vcpus < $currentvcpus) {
3782
3783         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3784
3785             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3786                 qemu_devicedel($vmid, "cpu$i");
3787                 my $retry = 0;
3788                 my $currentrunningvcpus = undef;
3789                 while (1) {
3790                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3791                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3792                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3793                     $retry++;
3794                     sleep 1;
3795                 }
3796                 #update conf after each succesfull cpu unplug
3797                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3798                 PVE::QemuConfig->write_config($vmid, $conf);
3799             }
3800         } else {
3801             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3802         }
3803
3804         return;
3805     }
3806
3807     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3808     die "vcpus in running vm does not match its configuration\n"
3809         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3810
3811     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3812
3813         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3814             my $cpustr = print_cpu_device($conf, $i);
3815             qemu_deviceadd($vmid, $cpustr);
3816
3817             my $retry = 0;
3818             my $currentrunningvcpus = undef;
3819             while (1) {
3820                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3821                 last if scalar(@{$currentrunningvcpus}) == $i;
3822                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3823                 sleep 1;
3824                 $retry++;
3825             }
3826             #update conf after each succesfull cpu hotplug
3827             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3828             PVE::QemuConfig->write_config($vmid, $conf);
3829         }
3830     } else {
3831
3832         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3833             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3834         }
3835     }
3836 }
3837
3838 sub qemu_block_set_io_throttle {
3839     my ($vmid, $deviceid,
3840         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3841         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3842         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3843         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3844
3845     return if !check_running($vmid) ;
3846
3847     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3848         bps => int($bps),
3849         bps_rd => int($bps_rd),
3850         bps_wr => int($bps_wr),
3851         iops => int($iops),
3852         iops_rd => int($iops_rd),
3853         iops_wr => int($iops_wr),
3854         bps_max => int($bps_max),
3855         bps_rd_max => int($bps_rd_max),
3856         bps_wr_max => int($bps_wr_max),
3857         iops_max => int($iops_max),
3858         iops_rd_max => int($iops_rd_max),
3859         iops_wr_max => int($iops_wr_max),
3860         bps_max_length => int($bps_max_length),
3861         bps_rd_max_length => int($bps_rd_max_length),
3862         bps_wr_max_length => int($bps_wr_max_length),
3863         iops_max_length => int($iops_max_length),
3864         iops_rd_max_length => int($iops_rd_max_length),
3865         iops_wr_max_length => int($iops_wr_max_length),
3866     );
3867
3868 }
3869
3870 # old code, only used to shutdown old VM after update
3871 sub __read_avail {
3872     my ($fh, $timeout) = @_;
3873
3874     my $sel = new IO::Select;
3875     $sel->add($fh);
3876
3877     my $res = '';
3878     my $buf;
3879
3880     my @ready;
3881     while (scalar (@ready = $sel->can_read($timeout))) {
3882         my $count;
3883         if ($count = $fh->sysread($buf, 8192)) {
3884             if ($buf =~ /^(.*)\(qemu\) $/s) {
3885                 $res .= $1;
3886                 last;
3887             } else {
3888                 $res .= $buf;
3889             }
3890         } else {
3891             if (!defined($count)) {
3892                 die "$!\n";
3893             }
3894             last;
3895         }
3896     }
3897
3898     die "monitor read timeout\n" if !scalar(@ready);
3899
3900     return $res;
3901 }
3902
3903 # old code, only used to shutdown old VM after update
3904 sub vm_monitor_command {
3905     my ($vmid, $cmdstr, $nocheck) = @_;
3906
3907     my $res;
3908
3909     eval {
3910         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3911
3912         my $sname = "${var_run_tmpdir}/$vmid.mon";
3913
3914         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3915             die "unable to connect to VM $vmid socket - $!\n";
3916
3917         my $timeout = 3;
3918
3919         # hack: migrate sometime blocks the monitor (when migrate_downtime
3920         # is set)
3921         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3922             $timeout = 60*60; # 1 hour
3923         }
3924
3925         # read banner;
3926         my $data = __read_avail($sock, $timeout);
3927
3928         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3929             die "got unexpected qemu monitor banner\n";
3930         }
3931
3932         my $sel = new IO::Select;
3933         $sel->add($sock);
3934
3935         if (!scalar(my @ready = $sel->can_write($timeout))) {
3936             die "monitor write error - timeout";
3937         }
3938
3939         my $fullcmd = "$cmdstr\r";
3940
3941         # syslog('info', "VM $vmid monitor command: $cmdstr");
3942
3943         my $b;
3944         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3945             die "monitor write error - $!";
3946         }
3947
3948         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
3949
3950         $timeout = 20;
3951
3952         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3953             $timeout = 60*60; # 1 hour
3954         } elsif ($cmdstr =~ m/^(eject|change)/) {
3955             $timeout = 60; # note: cdrom mount command is slow
3956         }
3957         if ($res = __read_avail($sock, $timeout)) {
3958
3959             my @lines = split("\r?\n", $res);
3960
3961             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
3962
3963             $res = join("\n", @lines);
3964             $res .= "\n";
3965         }
3966     };
3967
3968     my $err = $@;
3969
3970     if ($err) {
3971         syslog("err", "VM $vmid monitor command failed - $err");
3972         die $err;
3973     }
3974
3975     return $res;
3976 }
3977
3978 sub qemu_block_resize {
3979     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
3980
3981     my $running = check_running($vmid);
3982
3983     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
3984
3985     return if !$running;
3986
3987     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
3988
3989 }
3990
3991 sub qemu_volume_snapshot {
3992     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
3993
3994     my $running = check_running($vmid);
3995
3996     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
3997         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
3998     } else {
3999         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4000     }
4001 }
4002
4003 sub qemu_volume_snapshot_delete {
4004     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4005
4006     my $running = check_running($vmid);
4007
4008     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4009         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4010     } else {
4011         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4012     }
4013 }
4014
4015 sub set_migration_caps {
4016     my ($vmid) = @_;
4017
4018     my $cap_ref = [];
4019
4020     my $enabled_cap = {
4021         "auto-converge" => 1,
4022         "xbzrle" => 1,
4023         "x-rdma-pin-all" => 0,
4024         "zero-blocks" => 0,
4025         "compress" => 0
4026     };
4027
4028     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4029
4030     for my $supported_capability (@$supported_capabilities) {
4031         push @$cap_ref, {
4032             capability => $supported_capability->{capability},
4033             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4034         };
4035     }
4036
4037     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4038 }
4039
4040 my $fast_plug_option = {
4041     'lock' => 1,
4042     'name' => 1,
4043     'onboot' => 1,
4044     'shares' => 1,
4045     'startup' => 1,
4046     'description' => 1,
4047     'protection' => 1,
4048 };
4049
4050 # hotplug changes in [PENDING]
4051 # $selection hash can be used to only apply specified options, for
4052 # example: { cores => 1 } (only apply changed 'cores')
4053 # $errors ref is used to return error messages
4054 sub vmconfig_hotplug_pending {
4055     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4056
4057     my $defaults = load_defaults();
4058
4059     # commit values which do not have any impact on running VM first
4060     # Note: those option cannot raise errors, we we do not care about
4061     # $selection and always apply them.
4062
4063     my $add_error = sub {
4064         my ($opt, $msg) = @_;
4065         $errors->{$opt} = "hotplug problem - $msg";
4066     };
4067
4068     my $changes = 0;
4069     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4070         if ($fast_plug_option->{$opt}) {
4071             $conf->{$opt} = $conf->{pending}->{$opt};
4072             delete $conf->{pending}->{$opt};
4073             $changes = 1;
4074         }
4075     }
4076
4077     if ($changes) {
4078         PVE::QemuConfig->write_config($vmid, $conf);
4079         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4080     }
4081
4082     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4083
4084     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4085     while (my ($opt, $force) = each %$pending_delete_hash) {
4086         next if $selection && !$selection->{$opt};
4087         eval {
4088             if ($opt eq 'hotplug') {
4089                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4090             } elsif ($opt eq 'tablet') {
4091                 die "skip\n" if !$hotplug_features->{usb};
4092                 if ($defaults->{tablet}) {
4093                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4094                 } else {
4095                     vm_deviceunplug($vmid, $conf, $opt);
4096                 }
4097             } elsif ($opt =~ m/^usb\d+/) {
4098                 die "skip\n";
4099                 # since we cannot reliably hot unplug usb devices
4100                 # we are disabling it
4101                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4102                 vm_deviceunplug($vmid, $conf, $opt);
4103             } elsif ($opt eq 'vcpus') {
4104                 die "skip\n" if !$hotplug_features->{cpu};
4105                 qemu_cpu_hotplug($vmid, $conf, undef);
4106             } elsif ($opt eq 'balloon') {
4107                 # enable balloon device is not hotpluggable
4108                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4109             } elsif ($fast_plug_option->{$opt}) {
4110                 # do nothing
4111             } elsif ($opt =~ m/^net(\d+)$/) {
4112                 die "skip\n" if !$hotplug_features->{network};
4113                 vm_deviceunplug($vmid, $conf, $opt);
4114             } elsif (is_valid_drivename($opt)) {
4115                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4116                 vm_deviceunplug($vmid, $conf, $opt);
4117                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4118             } elsif ($opt =~ m/^memory$/) {
4119                 die "skip\n" if !$hotplug_features->{memory};
4120                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4121             } elsif ($opt eq 'cpuunits') {
4122                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4123             } elsif ($opt eq 'cpulimit') {
4124                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4125             } else {
4126                 die "skip\n";
4127             }
4128         };
4129         if (my $err = $@) {
4130             &$add_error($opt, $err) if $err ne "skip\n";
4131         } else {
4132             # save new config if hotplug was successful
4133             delete $conf->{$opt};
4134             vmconfig_undelete_pending_option($conf, $opt);
4135             PVE::QemuConfig->write_config($vmid, $conf);
4136             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4137         }
4138     }
4139
4140     foreach my $opt (keys %{$conf->{pending}}) {
4141         next if $selection && !$selection->{$opt};
4142         my $value = $conf->{pending}->{$opt};
4143         eval {
4144             if ($opt eq 'hotplug') {
4145                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4146             } elsif ($opt eq 'tablet') {
4147                 die "skip\n" if !$hotplug_features->{usb};
4148                 if ($value == 1) {
4149                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4150                 } elsif ($value == 0) {
4151                     vm_deviceunplug($vmid, $conf, $opt);
4152                 }
4153             } elsif ($opt =~ m/^usb\d+$/) {
4154                 die "skip\n";
4155                 # since we cannot reliably hot unplug usb devices
4156                 # we are disabling it
4157                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4158                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4159                 die "skip\n" if !$d;
4160                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4161             } elsif ($opt eq 'vcpus') {
4162                 die "skip\n" if !$hotplug_features->{cpu};
4163                 qemu_cpu_hotplug($vmid, $conf, $value);
4164             } elsif ($opt eq 'balloon') {
4165                 # enable/disable balloning device is not hotpluggable
4166                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4167                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4168                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4169
4170                 # allow manual ballooning if shares is set to zero
4171                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4172                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4173                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4174                 }
4175             } elsif ($opt =~ m/^net(\d+)$/) {
4176                 # some changes can be done without hotplug
4177                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4178                                     $vmid, $opt, $value);
4179             } elsif (is_valid_drivename($opt)) {
4180                 # some changes can be done without hotplug
4181                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4182                                      $vmid, $opt, $value, 1);
4183             } elsif ($opt =~ m/^memory$/) { #dimms
4184                 die "skip\n" if !$hotplug_features->{memory};
4185                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4186             } elsif ($opt eq 'cpuunits') {
4187                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4188             } elsif ($opt eq 'cpulimit') {
4189                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4190                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4191             } else {
4192                 die "skip\n";  # skip non-hot-pluggable options
4193             }
4194         };
4195         if (my $err = $@) {
4196             &$add_error($opt, $err) if $err ne "skip\n";
4197         } else {
4198             # save new config if hotplug was successful
4199             $conf->{$opt} = $value;
4200             delete $conf->{pending}->{$opt};
4201             PVE::QemuConfig->write_config($vmid, $conf);
4202             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4203         }
4204     }
4205 }
4206
4207 sub try_deallocate_drive {
4208     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4209
4210     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4211         my $volid = $drive->{file};
4212         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4213             my $sid = PVE::Storage::parse_volume_id($volid);
4214             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4215
4216             # check if the disk is really unused
4217             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4218                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4219             PVE::Storage::vdisk_free($storecfg, $volid);
4220             return 1;
4221         } else {
4222             # If vm is not owner of this disk remove from config
4223             return 1;
4224         }
4225     }
4226
4227     return undef;
4228 }
4229
4230 sub vmconfig_delete_or_detach_drive {
4231     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4232
4233     my $drive = parse_drive($opt, $conf->{$opt});
4234
4235     my $rpcenv = PVE::RPCEnvironment::get();
4236     my $authuser = $rpcenv->get_user();
4237
4238     if ($force) {
4239         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4240         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4241     } else {
4242         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4243     }
4244 }
4245
4246 sub vmconfig_apply_pending {
4247     my ($vmid, $conf, $storecfg) = @_;
4248
4249     # cold plug
4250
4251     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4252     while (my ($opt, $force) = each %$pending_delete_hash) {
4253         die "internal error" if $opt =~ m/^unused/;
4254         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4255         if (!defined($conf->{$opt})) {
4256             vmconfig_undelete_pending_option($conf, $opt);
4257             PVE::QemuConfig->write_config($vmid, $conf);
4258         } elsif (is_valid_drivename($opt)) {
4259             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4260             vmconfig_undelete_pending_option($conf, $opt);
4261             delete $conf->{$opt};
4262             PVE::QemuConfig->write_config($vmid, $conf);
4263         } else {
4264             vmconfig_undelete_pending_option($conf, $opt);
4265             delete $conf->{$opt};
4266             PVE::QemuConfig->write_config($vmid, $conf);
4267         }
4268     }
4269
4270     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4271
4272     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4273         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4274
4275         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4276             # skip if nothing changed
4277         } elsif (is_valid_drivename($opt)) {
4278             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4279                 if defined($conf->{$opt});
4280             $conf->{$opt} = $conf->{pending}->{$opt};
4281         } else {
4282             $conf->{$opt} = $conf->{pending}->{$opt};
4283         }
4284
4285         delete $conf->{pending}->{$opt};
4286         PVE::QemuConfig->write_config($vmid, $conf);
4287     }
4288 }
4289
4290 my $safe_num_ne = sub {
4291     my ($a, $b) = @_;
4292
4293     return 0 if !defined($a) && !defined($b);
4294     return 1 if !defined($a);
4295     return 1 if !defined($b);
4296
4297     return $a != $b;
4298 };
4299
4300 my $safe_string_ne = sub {
4301     my ($a, $b) = @_;
4302
4303     return 0 if !defined($a) && !defined($b);
4304     return 1 if !defined($a);
4305     return 1 if !defined($b);
4306
4307     return $a ne $b;
4308 };
4309
4310 sub vmconfig_update_net {
4311     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4312
4313     my $newnet = parse_net($value);
4314
4315     if ($conf->{$opt}) {
4316         my $oldnet = parse_net($conf->{$opt});
4317
4318         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4319             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4320             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4321             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4322
4323             # for non online change, we try to hot-unplug
4324             die "skip\n" if !$hotplug;
4325             vm_deviceunplug($vmid, $conf, $opt);
4326         } else {
4327
4328             die "internal error" if $opt !~ m/net(\d+)/;
4329             my $iface = "tap${vmid}i$1";
4330
4331             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4332                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4333                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4334                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4335                 PVE::Network::tap_unplug($iface);
4336                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4337             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4338                 # Rate can be applied on its own but any change above needs to
4339                 # include the rate in tap_plug since OVS resets everything.
4340                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4341             }
4342
4343             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4344                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4345             }
4346
4347             return 1;
4348         }
4349     }
4350
4351     if ($hotplug) {
4352         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4353     } else {
4354         die "skip\n";
4355     }
4356 }
4357
4358 sub vmconfig_update_disk {
4359     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4360
4361     # fixme: do we need force?
4362
4363     my $drive = parse_drive($opt, $value);
4364
4365     if ($conf->{$opt}) {
4366
4367         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4368
4369             my $media = $drive->{media} || 'disk';
4370             my $oldmedia = $old_drive->{media} || 'disk';
4371             die "unable to change media type\n" if $media ne $oldmedia;
4372
4373             if (!drive_is_cdrom($old_drive)) {
4374
4375                 if ($drive->{file} ne $old_drive->{file}) {
4376
4377                     die "skip\n" if !$hotplug;
4378
4379                     # unplug and register as unused
4380                     vm_deviceunplug($vmid, $conf, $opt);
4381                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4382
4383                 } else {
4384                     # update existing disk
4385
4386                     # skip non hotpluggable value
4387                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4388                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4389                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4390                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4391                         die "skip\n";
4392                     }
4393
4394                     # apply throttle
4395                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4396                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4397                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4398                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4399                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4400                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4401                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4402                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4403                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4404                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4405                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4406                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4407                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4408                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4409                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4410                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4411                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4412                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4413
4414                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4415                                                    ($drive->{mbps} || 0)*1024*1024,
4416                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4417                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4418                                                    $drive->{iops} || 0,
4419                                                    $drive->{iops_rd} || 0,
4420                                                    $drive->{iops_wr} || 0,
4421                                                    ($drive->{mbps_max} || 0)*1024*1024,
4422                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4423                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4424                                                    $drive->{iops_max} || 0,
4425                                                    $drive->{iops_rd_max} || 0,
4426                                                    $drive->{iops_wr_max} || 0,
4427                                                    $drive->{bps_max_length} || 1,
4428                                                    $drive->{bps_rd_max_length} || 1,
4429                                                    $drive->{bps_wr_max_length} || 1,
4430                                                    $drive->{iops_max_length} || 1,
4431                                                    $drive->{iops_rd_max_length} || 1,
4432                                                    $drive->{iops_wr_max_length} || 1);
4433
4434                     }
4435
4436                     return 1;
4437                 }
4438
4439             } else { # cdrom
4440
4441                 if ($drive->{file} eq 'none') {
4442                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4443                 } else {
4444                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4445                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4446                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4447                 }
4448
4449                 return 1;
4450             }
4451         }
4452     }
4453
4454     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4455     # hotplug new disks
4456     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4457     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4458 }
4459
4460 sub vm_start {
4461     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4462         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4463
4464     PVE::QemuConfig->lock_config($vmid, sub {
4465         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4466
4467         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4468
4469         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4470
4471         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4472
4473         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4474             vmconfig_apply_pending($vmid, $conf, $storecfg);
4475             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4476         }
4477
4478         my $defaults = load_defaults();
4479
4480         # set environment variable useful inside network script
4481         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4482
4483         my $local_volumes = {};
4484
4485         if ($targetstorage) {
4486             foreach_drive($conf, sub {
4487                 my ($ds, $drive) = @_;
4488
4489                 return if drive_is_cdrom($drive);
4490
4491                 my $volid = $drive->{file};
4492
4493                 return if !$volid;
4494
4495                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4496
4497                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4498                 return if $scfg->{shared};
4499                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4500             });
4501
4502             my $format = undef;
4503
4504             foreach my $opt (sort keys %$local_volumes) {
4505
4506                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4507                 my $drive = parse_drive($opt, $conf->{$opt});
4508
4509                 #if remote storage is specified, use default format
4510                 if ($targetstorage && $targetstorage ne "1") {
4511                     $storeid = $targetstorage;
4512                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4513                     $format = $defFormat;
4514                 } else {
4515                     #else we use same format than original
4516                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4517                     $format = qemu_img_format($scfg, $volid);
4518                 }
4519
4520                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4521                 my $newdrive = $drive;
4522                 $newdrive->{format} = $format;
4523                 $newdrive->{file} = $newvolid;
4524                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4525                 $local_volumes->{$opt} = $drivestr;
4526                 #pass drive to conf for command line
4527                 $conf->{$opt} = $drivestr;
4528             }
4529         }
4530
4531         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4532
4533         my $migrate_port = 0;
4534         my $migrate_uri;
4535         if ($statefile) {
4536             if ($statefile eq 'tcp') {
4537                 my $localip = "localhost";
4538                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4539                 my $nodename = PVE::INotify::nodename();
4540
4541                 if (!defined($migration_type)) {
4542                     if (defined($datacenterconf->{migration}->{type})) {
4543                         $migration_type = $datacenterconf->{migration}->{type};
4544                     } else {
4545                         $migration_type = 'secure';
4546                     }
4547                 }
4548
4549                 if ($migration_type eq 'insecure') {
4550                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4551                     if ($migrate_network_addr) {
4552                         $localip = $migrate_network_addr;
4553                     } else {
4554                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4555                     }
4556
4557                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4558                 }
4559
4560                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4561                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4562                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4563                 push @$cmd, '-incoming', $migrate_uri;
4564                 push @$cmd, '-S';
4565
4566             } elsif ($statefile eq 'unix') {
4567                 # should be default for secure migrations as a ssh TCP forward
4568                 # tunnel is not deterministic reliable ready and fails regurarly
4569                 # to set up in time, so use UNIX socket forwards
4570                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4571                 unlink $socket_addr;
4572
4573                 $migrate_uri = "unix:$socket_addr";
4574
4575                 push @$cmd, '-incoming', $migrate_uri;
4576                 push @$cmd, '-S';
4577
4578             } else {
4579                 push @$cmd, '-loadstate', $statefile;
4580             }
4581         } elsif ($paused) {
4582             push @$cmd, '-S';
4583         }
4584
4585         # host pci devices
4586         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4587           my $d = parse_hostpci($conf->{"hostpci$i"});
4588           next if !$d;
4589           my $pcidevices = $d->{pciid};
4590           foreach my $pcidevice (@$pcidevices) {
4591                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4592
4593                 my $info = pci_device_info("0000:$pciid");
4594                 die "IOMMU not present\n" if !check_iommu_support();
4595                 die "no pci device info for device '$pciid'\n" if !$info;
4596                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4597                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4598           }
4599         }
4600
4601         PVE::Storage::activate_volumes($storecfg, $vollist);
4602
4603         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4604             my $cmd = [];
4605             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4606             eval  { run_command($cmd); };
4607         }
4608
4609         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4610                                                   : $defaults->{cpuunits};
4611
4612         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4613
4614         my %properties = (
4615             Slice => 'qemu.slice',
4616             KillMode => 'none',
4617             CPUShares => $cpuunits
4618         );
4619
4620         if (my $cpulimit = $conf->{cpulimit}) {
4621             $properties{CPUQuota} = int($cpulimit * 100);
4622         }
4623         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4624
4625         if ($conf->{hugepages}) {
4626
4627             my $code = sub {
4628                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4629                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4630
4631                 PVE::QemuServer::Memory::hugepages_mount();
4632                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4633
4634                 eval  {
4635                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4636                     run_command($cmd, %run_params);
4637                 };
4638
4639                 if (my $err = $@) {
4640                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4641                     die $err;
4642                 }
4643
4644                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4645             };
4646             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4647
4648         } else {
4649             eval  {
4650                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4651                 run_command($cmd, %run_params);
4652             };
4653         }
4654
4655         if (my $err = $@) {
4656             # deactivate volumes if start fails
4657             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4658             die "start failed: $err";
4659         }
4660
4661         print "migration listens on $migrate_uri\n" if $migrate_uri;
4662
4663         if ($statefile && $statefile ne 'tcp')  {
4664             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4665             warn $@ if $@;
4666         }
4667
4668         #start nbd server for storage migration
4669         if ($targetstorage) {
4670             my $nodename = PVE::INotify::nodename();
4671             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4672             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4673             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4674             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4675
4676             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4677
4678             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4679
4680             foreach my $opt (sort keys %$local_volumes) {
4681                 my $volid = $local_volumes->{$opt};
4682                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4683                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4684                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4685             }
4686         }
4687
4688         if ($migratedfrom) {
4689             eval {
4690                 set_migration_caps($vmid);
4691             };
4692             warn $@ if $@;
4693
4694             if ($spice_port) {
4695                 print "spice listens on port $spice_port\n";
4696                 if ($spice_ticket) {
4697                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4698                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4699                 }
4700             }
4701
4702         } else {
4703             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4704                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4705                     if $conf->{balloon};
4706             }
4707
4708             foreach my $opt (keys %$conf) {
4709                 next if $opt !~  m/^net\d+$/;
4710                 my $nicconf = parse_net($conf->{$opt});
4711                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4712             }
4713         }
4714
4715         vm_mon_cmd_nocheck($vmid, 'qom-set',
4716                     path => "machine/peripheral/balloon0",
4717                     property => "guest-stats-polling-interval",
4718                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4719
4720     });
4721 }
4722
4723 sub vm_mon_cmd {
4724     my ($vmid, $execute, %params) = @_;
4725
4726     my $cmd = { execute => $execute, arguments => \%params };
4727     vm_qmp_command($vmid, $cmd);
4728 }
4729
4730 sub vm_mon_cmd_nocheck {
4731     my ($vmid, $execute, %params) = @_;
4732
4733     my $cmd = { execute => $execute, arguments => \%params };
4734     vm_qmp_command($vmid, $cmd, 1);
4735 }
4736
4737 sub vm_qmp_command {
4738     my ($vmid, $cmd, $nocheck) = @_;
4739
4740     my $res;
4741
4742     my $timeout;
4743     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4744         $timeout = $cmd->{arguments}->{timeout};
4745         delete $cmd->{arguments}->{timeout};
4746     }
4747
4748     eval {
4749         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4750         my $sname = qmp_socket($vmid);
4751         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4752             my $qmpclient = PVE::QMPClient->new();
4753
4754             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4755         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4756             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4757                 if scalar(%{$cmd->{arguments}});
4758             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4759         } else {
4760             die "unable to open monitor socket\n";
4761         }
4762     };
4763     if (my $err = $@) {
4764         syslog("err", "VM $vmid qmp command failed - $err");
4765         die $err;
4766     }
4767
4768     return $res;
4769 }
4770
4771 sub vm_human_monitor_command {
4772     my ($vmid, $cmdline) = @_;
4773
4774     my $res;
4775
4776     my $cmd = {
4777         execute => 'human-monitor-command',
4778         arguments => { 'command-line' => $cmdline},
4779     };
4780
4781     return vm_qmp_command($vmid, $cmd);
4782 }
4783
4784 sub vm_commandline {
4785     my ($storecfg, $vmid) = @_;
4786
4787     my $conf = PVE::QemuConfig->load_config($vmid);
4788
4789     my $defaults = load_defaults();
4790
4791     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4792
4793     return PVE::Tools::cmd2string($cmd);
4794 }
4795
4796 sub vm_reset {
4797     my ($vmid, $skiplock) = @_;
4798
4799     PVE::QemuConfig->lock_config($vmid, sub {
4800
4801         my $conf = PVE::QemuConfig->load_config($vmid);
4802
4803         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4804
4805         vm_mon_cmd($vmid, "system_reset");
4806     });
4807 }
4808
4809 sub get_vm_volumes {
4810     my ($conf) = @_;
4811
4812     my $vollist = [];
4813     foreach_volid($conf, sub {
4814         my ($volid, $is_cdrom) = @_;
4815
4816         return if $volid =~ m|^/|;
4817
4818         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4819         return if !$sid;
4820
4821         push @$vollist, $volid;
4822     });
4823
4824     return $vollist;
4825 }
4826
4827 sub vm_stop_cleanup {
4828     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4829
4830     eval {
4831
4832         if (!$keepActive) {
4833             my $vollist = get_vm_volumes($conf);
4834             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4835         }
4836
4837         foreach my $ext (qw(mon qmp pid vnc qga)) {
4838             unlink "/var/run/qemu-server/${vmid}.$ext";
4839         }
4840
4841         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4842     };
4843     warn $@ if $@; # avoid errors - just warn
4844 }
4845
4846 # Note: use $nockeck to skip tests if VM configuration file exists.
4847 # We need that when migration VMs to other nodes (files already moved)
4848 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4849 sub vm_stop {
4850     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4851
4852     $force = 1 if !defined($force) && !$shutdown;
4853
4854     if ($migratedfrom){
4855         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4856         kill 15, $pid if $pid;
4857         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4858         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4859         return;
4860     }
4861
4862     PVE::QemuConfig->lock_config($vmid, sub {
4863
4864         my $pid = check_running($vmid, $nocheck);
4865         return if !$pid;
4866
4867         my $conf;
4868         if (!$nocheck) {
4869             $conf = PVE::QemuConfig->load_config($vmid);
4870             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4871             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4872                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4873                 $timeout = $opts->{down} if $opts->{down};
4874             }
4875         }
4876
4877         $timeout = 60 if !defined($timeout);
4878
4879         eval {
4880             if ($shutdown) {
4881                 if (defined($conf) && $conf->{agent}) {
4882                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4883                 } else {
4884                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4885                 }
4886             } else {
4887                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4888             }
4889         };
4890         my $err = $@;
4891
4892         if (!$err) {
4893             my $count = 0;
4894             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4895                 $count++;
4896                 sleep 1;
4897             }
4898
4899             if ($count >= $timeout) {
4900                 if ($force) {
4901                     warn "VM still running - terminating now with SIGTERM\n";
4902                     kill 15, $pid;
4903                 } else {
4904                     die "VM quit/powerdown failed - got timeout\n";
4905                 }
4906             } else {
4907                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4908                 return;
4909             }
4910         } else {
4911             if ($force) {
4912                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4913                 kill 15, $pid;
4914             } else {
4915                 die "VM quit/powerdown failed\n";
4916             }
4917         }
4918
4919         # wait again
4920         $timeout = 10;
4921
4922         my $count = 0;
4923         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4924             $count++;
4925             sleep 1;
4926         }
4927
4928         if ($count >= $timeout) {
4929             warn "VM still running - terminating now with SIGKILL\n";
4930             kill 9, $pid;
4931             sleep 1;
4932         }
4933
4934         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4935    });
4936 }
4937
4938 sub vm_suspend {
4939     my ($vmid, $skiplock) = @_;
4940
4941     PVE::QemuConfig->lock_config($vmid, sub {
4942
4943         my $conf = PVE::QemuConfig->load_config($vmid);
4944
4945         PVE::QemuConfig->check_lock($conf)
4946             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4947
4948         vm_mon_cmd($vmid, "stop");
4949     });
4950 }
4951
4952 sub vm_resume {
4953     my ($vmid, $skiplock, $nocheck) = @_;
4954
4955     PVE::QemuConfig->lock_config($vmid, sub {
4956
4957         if (!$nocheck) {
4958
4959             my $conf = PVE::QemuConfig->load_config($vmid);
4960
4961             PVE::QemuConfig->check_lock($conf)
4962                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4963
4964             vm_mon_cmd($vmid, "cont");
4965
4966         } else {
4967             vm_mon_cmd_nocheck($vmid, "cont");
4968         }
4969     });
4970 }
4971
4972 sub vm_sendkey {
4973     my ($vmid, $skiplock, $key) = @_;
4974
4975     PVE::QemuConfig->lock_config($vmid, sub {
4976
4977         my $conf = PVE::QemuConfig->load_config($vmid);
4978
4979         # there is no qmp command, so we use the human monitor command
4980         vm_human_monitor_command($vmid, "sendkey $key");
4981     });
4982 }
4983
4984 sub vm_destroy {
4985     my ($storecfg, $vmid, $skiplock) = @_;
4986
4987     PVE::QemuConfig->lock_config($vmid, sub {
4988
4989         my $conf = PVE::QemuConfig->load_config($vmid);
4990
4991         if (!check_running($vmid)) {
4992             destroy_vm($storecfg, $vmid, undef, $skiplock);
4993         } else {
4994             die "VM $vmid is running - destroy failed\n";
4995         }
4996     });
4997 }
4998
4999 # pci helpers
5000
5001 sub file_write {
5002     my ($filename, $buf) = @_;
5003
5004     my $fh = IO::File->new($filename, "w");
5005     return undef if !$fh;
5006
5007     my $res = print $fh $buf;
5008
5009     $fh->close();
5010
5011     return $res;
5012 }
5013
5014 sub pci_device_info {
5015     my ($name) = @_;
5016
5017     my $res;
5018
5019     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5020     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5021
5022     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5023     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5024
5025     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5026     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5027
5028     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5029     return undef if !defined($product) || $product !~ s/^0x//;
5030
5031     $res = {
5032         name => $name,
5033         vendor => $vendor,
5034         product => $product,
5035         domain => $domain,
5036         bus => $bus,
5037         slot => $slot,
5038         func => $func,
5039         irq => $irq,
5040         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5041     };
5042
5043     return $res;
5044 }
5045
5046 sub pci_dev_reset {
5047     my ($dev) = @_;
5048
5049     my $name = $dev->{name};
5050
5051     my $fn = "$pcisysfs/devices/$name/reset";
5052
5053     return file_write($fn, "1");
5054 }
5055
5056 sub pci_dev_bind_to_vfio {
5057     my ($dev) = @_;
5058
5059     my $name = $dev->{name};
5060
5061     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5062
5063     if (!-d $vfio_basedir) {
5064         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5065     }
5066     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5067
5068     my $testdir = "$vfio_basedir/$name";
5069     return 1 if -d $testdir;
5070
5071     my $data = "$dev->{vendor} $dev->{product}";
5072     return undef if !file_write("$vfio_basedir/new_id", $data);
5073
5074     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5075     if (!file_write($fn, $name)) {
5076         return undef if -f $fn;
5077     }
5078
5079     $fn = "$vfio_basedir/bind";
5080     if (! -d $testdir) {
5081         return undef if !file_write($fn, $name);
5082     }
5083
5084     return -d $testdir;
5085 }
5086
5087 sub pci_dev_group_bind_to_vfio {
5088     my ($pciid) = @_;
5089
5090     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5091
5092     if (!-d $vfio_basedir) {
5093         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5094     }
5095     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5096
5097     # get IOMMU group devices
5098     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5099       my @devs = grep /^0000:/, readdir($D);
5100     closedir($D);
5101
5102     foreach my $pciid (@devs) {
5103         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5104
5105         # pci bridges, switches or root ports are not supported
5106         # they have a pci_bus subdirectory so skip them
5107         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5108
5109         my $info = pci_device_info($1);
5110         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5111     }
5112
5113     return 1;
5114 }
5115
5116 # vzdump restore implementaion
5117
5118 sub tar_archive_read_firstfile {
5119     my $archive = shift;
5120
5121     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5122
5123     # try to detect archive type first
5124     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5125         die "unable to open file '$archive'\n";
5126     my $firstfile = <$fh>;
5127     kill 15, $pid;
5128     close $fh;
5129
5130     die "ERROR: archive contaions no data\n" if !$firstfile;
5131     chomp $firstfile;
5132
5133     return $firstfile;
5134 }
5135
5136 sub tar_restore_cleanup {
5137     my ($storecfg, $statfile) = @_;
5138
5139     print STDERR "starting cleanup\n";
5140
5141     if (my $fd = IO::File->new($statfile, "r")) {
5142         while (defined(my $line = <$fd>)) {
5143             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5144                 my $volid = $2;
5145                 eval {
5146                     if ($volid =~ m|^/|) {
5147                         unlink $volid || die 'unlink failed\n';
5148                     } else {
5149                         PVE::Storage::vdisk_free($storecfg, $volid);
5150                     }
5151                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5152                 };
5153                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5154             } else {
5155                 print STDERR "unable to parse line in statfile - $line";
5156             }
5157         }
5158         $fd->close();
5159     }
5160 }
5161
5162 sub restore_archive {
5163     my ($archive, $vmid, $user, $opts) = @_;
5164
5165     my $format = $opts->{format};
5166     my $comp;
5167
5168     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5169         $format = 'tar' if !$format;
5170         $comp = 'gzip';
5171     } elsif ($archive =~ m/\.tar$/) {
5172         $format = 'tar' if !$format;
5173     } elsif ($archive =~ m/.tar.lzo$/) {
5174         $format = 'tar' if !$format;
5175         $comp = 'lzop';
5176     } elsif ($archive =~ m/\.vma$/) {
5177         $format = 'vma' if !$format;
5178     } elsif ($archive =~ m/\.vma\.gz$/) {
5179         $format = 'vma' if !$format;
5180         $comp = 'gzip';
5181     } elsif ($archive =~ m/\.vma\.lzo$/) {
5182         $format = 'vma' if !$format;
5183         $comp = 'lzop';
5184     } else {
5185         $format = 'vma' if !$format; # default
5186     }
5187
5188     # try to detect archive format
5189     if ($format eq 'tar') {
5190         return restore_tar_archive($archive, $vmid, $user, $opts);
5191     } else {
5192         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5193     }
5194 }
5195
5196 sub restore_update_config_line {
5197     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5198
5199     return if $line =~ m/^\#qmdump\#/;
5200     return if $line =~ m/^\#vzdump\#/;
5201     return if $line =~ m/^lock:/;
5202     return if $line =~ m/^unused\d+:/;
5203     return if $line =~ m/^parent:/;
5204     return if $line =~ m/^template:/; # restored VM is never a template
5205
5206     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5207     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5208         # try to convert old 1.X settings
5209         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5210         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5211             my ($model, $macaddr) = split(/\=/, $devconfig);
5212             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5213             my $net = {
5214                 model => $model,
5215                 bridge => "vmbr$ind",
5216                 macaddr => $macaddr,
5217             };
5218             my $netstr = print_net($net);
5219
5220             print $outfd "net$cookie->{netcount}: $netstr\n";
5221             $cookie->{netcount}++;
5222         }
5223     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5224         my ($id, $netstr) = ($1, $2);
5225         my $net = parse_net($netstr);
5226         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5227         $netstr = print_net($net);
5228         print $outfd "$id: $netstr\n";
5229     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5230         my $virtdev = $1;
5231         my $value = $3;
5232         my $di = parse_drive($virtdev, $value);
5233         if (defined($di->{backup}) && !$di->{backup}) {
5234             print $outfd "#$line";
5235         } elsif ($map->{$virtdev}) {
5236             delete $di->{format}; # format can change on restore
5237             $di->{file} = $map->{$virtdev};
5238             $value = print_drive($vmid, $di);
5239             print $outfd "$virtdev: $value\n";
5240         } else {
5241             print $outfd $line;
5242         }
5243     } else {
5244         print $outfd $line;
5245     }
5246 }
5247
5248 sub scan_volids {
5249     my ($cfg, $vmid) = @_;
5250
5251     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5252
5253     my $volid_hash = {};
5254     foreach my $storeid (keys %$info) {
5255         foreach my $item (@{$info->{$storeid}}) {
5256             next if !($item->{volid} && $item->{size});
5257             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5258             $volid_hash->{$item->{volid}} = $item;
5259         }
5260     }
5261
5262     return $volid_hash;
5263 }
5264
5265 sub is_volume_in_use {
5266     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5267
5268     my $path = PVE::Storage::path($storecfg, $volid);
5269
5270     my $scan_config = sub {
5271         my ($cref, $snapname) = @_;
5272
5273         foreach my $key (keys %$cref) {
5274             my $value = $cref->{$key};
5275             if (is_valid_drivename($key)) {
5276                 next if $skip_drive && $key eq $skip_drive;
5277                 my $drive = parse_drive($key, $value);
5278                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5279                 return 1 if $volid eq $drive->{file};
5280                 if ($drive->{file} =~ m!^/!) {
5281                     return 1 if $drive->{file} eq $path;
5282                 } else {
5283                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5284                     next if !$storeid;
5285                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5286                     next if !$scfg;
5287                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5288                 }
5289             }
5290         }
5291
5292         return 0;
5293     };
5294
5295     return 1 if &$scan_config($conf);
5296
5297     undef $skip_drive;
5298
5299     foreach my $snapname (keys %{$conf->{snapshots}}) {
5300         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5301     }
5302
5303     return 0;
5304 }
5305
5306 sub update_disksize {
5307     my ($vmid, $conf, $volid_hash) = @_;
5308
5309     my $changes;
5310
5311     my $used = {};
5312
5313     # Note: it is allowed to define multiple storages with same path (alias), so
5314     # we need to check both 'volid' and real 'path' (two different volid can point
5315     # to the same path).
5316
5317     my $usedpath = {};
5318
5319     # update size info
5320     foreach my $opt (keys %$conf) {
5321         if (is_valid_drivename($opt)) {
5322             my $drive = parse_drive($opt, $conf->{$opt});
5323             my $volid = $drive->{file};
5324             next if !$volid;
5325
5326             $used->{$volid} = 1;
5327             if ($volid_hash->{$volid} &&
5328                 (my $path = $volid_hash->{$volid}->{path})) {
5329                 $usedpath->{$path} = 1;
5330             }
5331
5332             next if drive_is_cdrom($drive);
5333             next if !$volid_hash->{$volid};
5334
5335             $drive->{size} = $volid_hash->{$volid}->{size};
5336             my $new = print_drive($vmid, $drive);
5337             if ($new ne $conf->{$opt}) {
5338                 $changes = 1;
5339                 $conf->{$opt} = $new;
5340             }
5341         }
5342     }
5343
5344     # remove 'unusedX' entry if volume is used
5345     foreach my $opt (keys %$conf) {
5346         next if $opt !~ m/^unused\d+$/;
5347         my $volid = $conf->{$opt};
5348         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5349         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5350             $changes = 1;
5351             delete $conf->{$opt};
5352         }
5353     }
5354
5355     foreach my $volid (sort keys %$volid_hash) {
5356         next if $volid =~ m/vm-$vmid-state-/;
5357         next if $used->{$volid};
5358         my $path = $volid_hash->{$volid}->{path};
5359         next if !$path; # just to be sure
5360         next if $usedpath->{$path};
5361         $changes = 1;
5362         PVE::QemuConfig->add_unused_volume($conf, $volid);
5363         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5364     }
5365
5366     return $changes;
5367 }
5368
5369 sub rescan {
5370     my ($vmid, $nolock) = @_;
5371
5372     my $cfg = PVE::Storage::config();
5373
5374     my $volid_hash = scan_volids($cfg, $vmid);
5375
5376     my $updatefn =  sub {
5377         my ($vmid) = @_;
5378
5379         my $conf = PVE::QemuConfig->load_config($vmid);
5380
5381         PVE::QemuConfig->check_lock($conf);
5382
5383         my $vm_volids = {};
5384         foreach my $volid (keys %$volid_hash) {
5385             my $info = $volid_hash->{$volid};
5386             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5387         }
5388
5389         my $changes = update_disksize($vmid, $conf, $vm_volids);
5390
5391         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5392     };
5393
5394     if (defined($vmid)) {
5395         if ($nolock) {
5396             &$updatefn($vmid);
5397         } else {
5398             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5399         }
5400     } else {
5401         my $vmlist = config_list();
5402         foreach my $vmid (keys %$vmlist) {
5403             if ($nolock) {
5404                 &$updatefn($vmid);
5405             } else {
5406                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5407             }
5408         }
5409     }
5410 }
5411
5412 sub restore_vma_archive {
5413     my ($archive, $vmid, $user, $opts, $comp) = @_;
5414
5415     my $input = $archive eq '-' ? "<&STDIN" : undef;
5416     my $readfrom = $archive;
5417
5418     my $uncomp = '';
5419     if ($comp) {
5420         $readfrom = '-';
5421         my $qarchive = PVE::Tools::shellquote($archive);
5422         if ($comp eq 'gzip') {
5423             $uncomp = "zcat $qarchive|";
5424         } elsif ($comp eq 'lzop') {
5425             $uncomp = "lzop -d -c $qarchive|";
5426         } else {
5427             die "unknown compression method '$comp'\n";
5428         }
5429
5430     }
5431
5432     my $tmpdir = "/var/tmp/vzdumptmp$$";
5433     rmtree $tmpdir;
5434
5435     # disable interrupts (always do cleanups)
5436     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5437         warn "got interrupt - ignored\n";
5438     };
5439
5440     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5441     POSIX::mkfifo($mapfifo, 0600);
5442     my $fifofh;
5443
5444     my $openfifo = sub {
5445         open($fifofh, '>', $mapfifo) || die $!;
5446     };
5447
5448     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5449
5450     my $oldtimeout;
5451     my $timeout = 5;
5452
5453     my $devinfo = {};
5454
5455     my $rpcenv = PVE::RPCEnvironment::get();
5456
5457     my $conffile = PVE::QemuConfig->config_file($vmid);
5458     my $tmpfn = "$conffile.$$.tmp";
5459
5460     # Note: $oldconf is undef if VM does not exists
5461     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5462     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5463
5464     my $print_devmap = sub {
5465         my $virtdev_hash = {};
5466
5467         my $cfgfn = "$tmpdir/qemu-server.conf";
5468
5469         # we can read the config - that is already extracted
5470         my $fh = IO::File->new($cfgfn, "r") ||
5471             "unable to read qemu-server.conf - $!\n";
5472
5473         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5474         if (-f $fwcfgfn) {
5475             my $pve_firewall_dir = '/etc/pve/firewall';
5476             mkdir $pve_firewall_dir; # make sure the dir exists
5477             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5478         }
5479
5480         while (defined(my $line = <$fh>)) {
5481             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5482                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5483                 die "archive does not contain data for drive '$virtdev'\n"
5484                     if !$devinfo->{$devname};
5485                 if (defined($opts->{storage})) {
5486                     $storeid = $opts->{storage} || 'local';
5487                 } elsif (!$storeid) {
5488                     $storeid = 'local';
5489                 }
5490                 $format = 'raw' if !$format;
5491                 $devinfo->{$devname}->{devname} = $devname;
5492                 $devinfo->{$devname}->{virtdev} = $virtdev;
5493                 $devinfo->{$devname}->{format} = $format;
5494                 $devinfo->{$devname}->{storeid} = $storeid;
5495
5496                 # check permission on storage
5497                 my $pool = $opts->{pool}; # todo: do we need that?
5498                 if ($user ne 'root@pam') {
5499                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5500                 }
5501
5502                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5503             }
5504         }
5505
5506         foreach my $devname (keys %$devinfo) {
5507             die "found no device mapping information for device '$devname'\n"
5508                 if !$devinfo->{$devname}->{virtdev};
5509         }
5510
5511         my $cfg = PVE::Storage::config();
5512
5513         # create empty/temp config
5514         if ($oldconf) {
5515             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5516             foreach_drive($oldconf, sub {
5517                 my ($ds, $drive) = @_;
5518
5519                 return if drive_is_cdrom($drive);
5520
5521                 my $volid = $drive->{file};
5522
5523                 return if !$volid || $volid =~ m|^/|;
5524
5525                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5526                 return if !$path || !$owner || ($owner != $vmid);
5527
5528                 # Note: only delete disk we want to restore
5529                 # other volumes will become unused
5530                 if ($virtdev_hash->{$ds}) {
5531                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5532                     if (my $err = $@) {
5533                         warn $err;
5534                     }
5535                 }
5536             });
5537
5538             # delete vmstate files
5539             # since after the restore we have no snapshots anymore
5540             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5541                 my $snap = $oldconf->{snapshots}->{$snapname};
5542                 if ($snap->{vmstate}) {
5543                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5544                     if (my $err = $@) {
5545                         warn $err;
5546                     }
5547                 }
5548             }
5549         }
5550
5551         my $map = {};
5552         foreach my $virtdev (sort keys %$virtdev_hash) {
5553             my $d = $virtdev_hash->{$virtdev};
5554             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5555             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5556
5557             # test if requested format is supported
5558             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5559             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5560             $d->{format} = $defFormat if !$supported;
5561
5562             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5563                                                   $d->{format}, undef, $alloc_size);
5564             print STDERR "new volume ID is '$volid'\n";
5565             $d->{volid} = $volid;
5566             my $path = PVE::Storage::path($cfg, $volid);
5567
5568             PVE::Storage::activate_volumes($cfg,[$volid]);
5569
5570             my $write_zeros = 1;
5571             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5572                 $write_zeros = 0;
5573             }
5574
5575             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5576
5577             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5578             $map->{$virtdev} = $volid;
5579         }
5580
5581         $fh->seek(0, 0) || die "seek failed - $!\n";
5582
5583         my $outfd = new IO::File ($tmpfn, "w") ||
5584             die "unable to write config for VM $vmid\n";
5585
5586         my $cookie = { netcount => 0 };
5587         while (defined(my $line = <$fh>)) {
5588             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5589         }
5590
5591         $fh->close();
5592         $outfd->close();
5593     };
5594
5595     eval {
5596         # enable interrupts
5597         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5598             die "interrupted by signal\n";
5599         };
5600         local $SIG{ALRM} = sub { die "got timeout\n"; };
5601
5602         $oldtimeout = alarm($timeout);
5603
5604         my $parser = sub {
5605             my $line = shift;
5606
5607             print "$line\n";
5608
5609             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5610                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5611                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5612             } elsif ($line =~ m/^CTIME: /) {
5613                 # we correctly received the vma config, so we can disable
5614                 # the timeout now for disk allocation (set to 10 minutes, so
5615                 # that we always timeout if something goes wrong)
5616                 alarm(600);
5617                 &$print_devmap();
5618                 print $fifofh "done\n";
5619                 my $tmp = $oldtimeout || 0;
5620                 $oldtimeout = undef;
5621                 alarm($tmp);
5622                 close($fifofh);
5623             }
5624         };
5625
5626         print "restore vma archive: $cmd\n";
5627         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5628     };
5629     my $err = $@;
5630
5631     alarm($oldtimeout) if $oldtimeout;
5632
5633     my $vollist = [];
5634     foreach my $devname (keys %$devinfo) {
5635         my $volid = $devinfo->{$devname}->{volid};
5636         push @$vollist, $volid if $volid;
5637     }
5638
5639     my $cfg = PVE::Storage::config();
5640     PVE::Storage::deactivate_volumes($cfg, $vollist);
5641
5642     unlink $mapfifo;
5643
5644     if ($err) {
5645         rmtree $tmpdir;
5646         unlink $tmpfn;
5647
5648         foreach my $devname (keys %$devinfo) {
5649             my $volid = $devinfo->{$devname}->{volid};
5650             next if !$volid;
5651             eval {
5652                 if ($volid =~ m|^/|) {
5653                     unlink $volid || die 'unlink failed\n';
5654                 } else {
5655                     PVE::Storage::vdisk_free($cfg, $volid);
5656                 }
5657                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5658             };
5659             print STDERR "unable to cleanup '$volid' - $@" if $@;
5660         }
5661         die $err;
5662     }
5663
5664     rmtree $tmpdir;
5665
5666     rename($tmpfn, $conffile) ||
5667         die "unable to commit configuration file '$conffile'\n";
5668
5669     PVE::Cluster::cfs_update(); # make sure we read new file
5670
5671     eval { rescan($vmid, 1); };
5672     warn $@ if $@;
5673 }
5674
5675 sub restore_tar_archive {
5676     my ($archive, $vmid, $user, $opts) = @_;
5677
5678     if ($archive ne '-') {
5679         my $firstfile = tar_archive_read_firstfile($archive);
5680         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5681             if $firstfile ne 'qemu-server.conf';
5682     }
5683
5684     my $storecfg = PVE::Storage::config();
5685
5686     # destroy existing data - keep empty config
5687     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5688     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5689
5690     my $tocmd = "/usr/lib/qemu-server/qmextract";
5691
5692     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5693     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5694     $tocmd .= ' --prealloc' if $opts->{prealloc};
5695     $tocmd .= ' --info' if $opts->{info};
5696
5697     # tar option "xf" does not autodetect compression when read from STDIN,
5698     # so we pipe to zcat
5699     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5700         PVE::Tools::shellquote("--to-command=$tocmd");
5701
5702     my $tmpdir = "/var/tmp/vzdumptmp$$";
5703     mkpath $tmpdir;
5704
5705     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5706     local $ENV{VZDUMP_VMID} = $vmid;
5707     local $ENV{VZDUMP_USER} = $user;
5708
5709     my $conffile = PVE::QemuConfig->config_file($vmid);
5710     my $tmpfn = "$conffile.$$.tmp";
5711
5712     # disable interrupts (always do cleanups)
5713     local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = sub {
5714         print STDERR "got interrupt - ignored\n";
5715     };
5716
5717     eval {
5718         # enable interrupts
5719         local $SIG{INT} = $SIG{TERM} = $SIG{QUIT} = $SIG{HUP} = $SIG{PIPE} = sub {
5720             die "interrupted by signal\n";
5721         };
5722
5723         if ($archive eq '-') {
5724             print "extracting archive from STDIN\n";
5725             run_command($cmd, input => "<&STDIN");
5726         } else {
5727             print "extracting archive '$archive'\n";
5728             run_command($cmd);
5729         }
5730
5731         return if $opts->{info};
5732
5733         # read new mapping
5734         my $map = {};
5735         my $statfile = "$tmpdir/qmrestore.stat";
5736         if (my $fd = IO::File->new($statfile, "r")) {
5737             while (defined (my $line = <$fd>)) {
5738                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5739                     $map->{$1} = $2 if $1;
5740                 } else {
5741                     print STDERR "unable to parse line in statfile - $line\n";
5742                 }
5743             }
5744             $fd->close();
5745         }
5746
5747         my $confsrc = "$tmpdir/qemu-server.conf";
5748
5749         my $srcfd = new IO::File($confsrc, "r") ||
5750             die "unable to open file '$confsrc'\n";
5751
5752         my $outfd = new IO::File ($tmpfn, "w") ||
5753             die "unable to write config for VM $vmid\n";
5754
5755         my $cookie = { netcount => 0 };
5756         while (defined (my $line = <$srcfd>)) {
5757             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5758         }
5759
5760         $srcfd->close();
5761         $outfd->close();
5762     };
5763     my $err = $@;
5764
5765     if ($err) {
5766
5767         unlink $tmpfn;
5768
5769         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5770
5771         die $err;
5772     }
5773
5774     rmtree $tmpdir;
5775
5776     rename $tmpfn, $conffile ||
5777         die "unable to commit configuration file '$conffile'\n";
5778
5779     PVE::Cluster::cfs_update(); # make sure we read new file
5780
5781     eval { rescan($vmid, 1); };
5782     warn $@ if $@;
5783 };
5784
5785 sub foreach_writable_storage {
5786     my ($conf, $func) = @_;
5787
5788     my $sidhash = {};
5789
5790     foreach my $ds (keys %$conf) {
5791         next if !is_valid_drivename($ds);
5792
5793         my $drive = parse_drive($ds, $conf->{$ds});
5794         next if !$drive;
5795         next if drive_is_cdrom($drive);
5796
5797         my $volid = $drive->{file};
5798
5799         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5800         $sidhash->{$sid} = $sid if $sid;
5801     }
5802
5803     foreach my $sid (sort keys %$sidhash) {
5804         &$func($sid);
5805     }
5806 }
5807
5808 sub do_snapshots_with_qemu {
5809     my ($storecfg, $volid) = @_;
5810
5811     my $storage_name = PVE::Storage::parse_volume_id($volid);
5812
5813     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5814         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5815         return 1;
5816     }
5817
5818     if ($volid =~ m/\.(qcow2|qed)$/){
5819         return 1;
5820     }
5821
5822     return undef;
5823 }
5824
5825 sub qga_check_running {
5826     my ($vmid) = @_;
5827
5828     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5829     if ($@) {
5830         warn "Qemu Guest Agent are not running - $@";
5831         return 0;
5832     }
5833     return 1;
5834 }
5835
5836 sub template_create {
5837     my ($vmid, $conf, $disk) = @_;
5838
5839     my $storecfg = PVE::Storage::config();
5840
5841     foreach_drive($conf, sub {
5842         my ($ds, $drive) = @_;
5843
5844         return if drive_is_cdrom($drive);
5845         return if $disk && $ds ne $disk;
5846
5847         my $volid = $drive->{file};
5848         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5849
5850         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5851         $drive->{file} = $voliddst;
5852         $conf->{$ds} = print_drive($vmid, $drive);
5853         PVE::QemuConfig->write_config($vmid, $conf);
5854     });
5855 }
5856
5857 sub qemu_img_convert {
5858     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5859
5860     my $storecfg = PVE::Storage::config();
5861     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5862     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5863
5864     if ($src_storeid && $dst_storeid) {
5865
5866         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5867
5868         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5869         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5870
5871         my $src_format = qemu_img_format($src_scfg, $src_volname);
5872         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5873
5874         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5875         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5876
5877         my $cmd = [];
5878         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5879         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5880         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5881         if ($is_zero_initialized) {
5882             push @$cmd, "zeroinit:$dst_path";
5883         } else {
5884             push @$cmd, $dst_path;
5885         }
5886
5887         my $parser = sub {
5888             my $line = shift;
5889             if($line =~ m/\((\S+)\/100\%\)/){
5890                 my $percent = $1;
5891                 my $transferred = int($size * $percent / 100);
5892                 my $remaining = $size - $transferred;
5893
5894                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5895             }
5896
5897         };
5898
5899         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5900         my $err = $@;
5901         die "copy failed: $err" if $err;
5902     }
5903 }
5904
5905 sub qemu_img_format {
5906     my ($scfg, $volname) = @_;
5907
5908     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5909         return $1;
5910     } else {
5911         return "raw";
5912     }
5913 }
5914
5915 sub qemu_drive_mirror {
5916     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
5917
5918     $jobs = {} if !$jobs;
5919
5920     my $qemu_target;
5921     my $format;
5922     $jobs->{"drive-$drive"} = {};
5923
5924     if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
5925         my $server = $1;
5926         my $port = $2;
5927         my $exportname = $3;
5928
5929         $format = "nbd";
5930         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
5931         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
5932         my $cmd = ['socat', '-T30', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=5"];
5933
5934         my $pid = fork();
5935         if (!defined($pid)) {
5936             die "forking socat tunnel failed\n";
5937         } elsif ($pid == 0) {
5938             exec(@$cmd);
5939             warn "exec failed: $!\n";
5940             POSIX::_exit(-1);
5941         }
5942         $jobs->{"drive-$drive"}->{pid} = $pid;
5943
5944         my $timeout = 0;
5945         while (!-S $unixsocket) {
5946             die "nbd connection helper timed out\n"
5947                 if $timeout++ > 5;
5948             sleep 1;
5949         }
5950     } else {
5951         my $storecfg = PVE::Storage::config();
5952         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
5953
5954         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5955
5956         $format = qemu_img_format($dst_scfg, $dst_volname);
5957
5958         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5959
5960         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
5961     }
5962
5963     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
5964     $opts->{format} = $format if $format;
5965
5966     print "drive mirror is starting for drive-$drive\n";
5967
5968     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
5969
5970     if (my $err = $@) {
5971         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
5972         die "mirroring error: $err";
5973     }
5974
5975     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
5976 }
5977
5978 sub qemu_drive_mirror_monitor {
5979     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
5980
5981     eval {
5982         my $err_complete = 0;
5983
5984         while (1) {
5985             die "storage migration timed out\n" if $err_complete > 300;
5986
5987             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
5988
5989             my $running_mirror_jobs = {};
5990             foreach my $stat (@$stats) {
5991                 next if $stat->{type} ne 'mirror';
5992                 $running_mirror_jobs->{$stat->{device}} = $stat;
5993             }
5994
5995             my $readycounter = 0;
5996
5997             foreach my $job (keys %$jobs) {
5998
5999                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6000                     print "$job : finished\n";
6001                     delete $jobs->{$job};
6002                     next;
6003                 }
6004
6005                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6006
6007                 my $busy = $running_mirror_jobs->{$job}->{busy};
6008                 my $ready = $running_mirror_jobs->{$job}->{ready};
6009                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6010                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6011                     my $remaining = $total - $transferred;
6012                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6013
6014                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6015                 }
6016
6017                 $readycounter++ if $running_mirror_jobs->{$job}->{ready} eq 'true';
6018             }
6019
6020             last if scalar(keys %$jobs) == 0;
6021
6022             if ($readycounter == scalar(keys %$jobs)) {
6023                 print "all mirroring jobs are ready \n";
6024                 last if $skipcomplete; #do the complete later
6025
6026                 if ($vmiddst && $vmiddst != $vmid) {
6027                     if ($qga) {
6028                         print "freeze filesystem\n";
6029                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6030                     } else {
6031                         print "suspend vm\n";
6032                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6033                     }
6034
6035                     # if we clone a disk for a new target vm, we don't switch the disk
6036                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6037
6038                     if ($qga) {
6039                         print "unfreeze filesystem\n";
6040                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6041                     } else {
6042                         print "resume vm\n";
6043                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6044                     }
6045
6046                     last;
6047                 } else {
6048
6049                     foreach my $job (keys %$jobs) {
6050                         # try to switch the disk if source and destination are on the same guest
6051                         print "$job: Completing block job...\n";
6052
6053                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6054                         if ($@ =~ m/cannot be completed/) {
6055                             print "$job: Block job cannot be completed, try again.\n";
6056                             $err_complete++;
6057                         }else {
6058                             print "$job: Completed successfully.\n";
6059                             $jobs->{$job}->{complete} = 1;
6060                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6061                         }
6062                     }
6063                 }
6064             }
6065             sleep 1;
6066         }
6067     };
6068     my $err = $@;
6069
6070     if ($err) {
6071         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6072         die "mirroring error: $err";
6073     }
6074
6075 }
6076
6077 sub qemu_blockjobs_cancel {
6078     my ($vmid, $jobs) = @_;
6079
6080     foreach my $job (keys %$jobs) {
6081         print "$job: Cancelling block job\n";
6082         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6083         $jobs->{$job}->{cancel} = 1;
6084     }
6085
6086     while (1) {
6087         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6088
6089         my $running_jobs = {};
6090         foreach my $stat (@$stats) {
6091             $running_jobs->{$stat->{device}} = $stat;
6092         }
6093
6094         foreach my $job (keys %$jobs) {
6095
6096             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6097                 print "$job: Done.\n";
6098                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6099                 delete $jobs->{$job};
6100             }
6101         }
6102
6103         last if scalar(keys %$jobs) == 0;
6104
6105         sleep 1;
6106     }
6107 }
6108
6109 sub qemu_blockjobs_finish_tunnel {
6110    my ($vmid, $job, $cpid) = @_;
6111
6112    return if !$cpid;
6113
6114    for (my $i = 1; $i < 20; $i++) {
6115         my $waitpid = waitpid($cpid, WNOHANG);
6116         last if (defined($waitpid) && ($waitpid == $cpid));
6117
6118         if ($i == 10) {
6119             kill(15, $cpid);
6120         } elsif ($i >= 15) {
6121             kill(9, $cpid);
6122         }
6123         sleep (1);
6124     }
6125     unlink "/run/qemu-server/$vmid.mirror-$job";
6126 }
6127
6128 sub clone_disk {
6129     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6130         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6131
6132     my $newvolid;
6133
6134     if (!$full) {
6135         print "create linked clone of drive $drivename ($drive->{file})\n";
6136         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6137         push @$newvollist, $newvolid;
6138     } else {
6139
6140         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6141         $storeid = $storage if $storage;
6142
6143         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6144         if (!$format) {
6145             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6146             $format = qemu_img_format($scfg, $volname);
6147         }
6148
6149         # test if requested format is supported - else use default
6150         my $supported = grep { $_ eq $format } @$validFormats;
6151         $format = $defFormat if !$supported;
6152
6153         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6154
6155         print "create full clone of drive $drivename ($drive->{file})\n";
6156         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $format, undef, ($size/1024));
6157         push @$newvollist, $newvolid;
6158
6159         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6160
6161         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6162         if (!$running || $snapname) {
6163             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6164         } else {
6165
6166             my $kvmver = get_running_qemu_version ($vmid);
6167             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6168                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6169                     if $drive->{iothread};
6170             }
6171
6172             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6173         }
6174     }
6175
6176     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6177
6178     my $disk = $drive;
6179     $disk->{format} = undef;
6180     $disk->{file} = $newvolid;
6181     $disk->{size} = $size;
6182
6183     return $disk;
6184 }
6185
6186 # this only works if VM is running
6187 sub get_current_qemu_machine {
6188     my ($vmid) = @_;
6189
6190     my $cmd = { execute => 'query-machines', arguments => {} };
6191     my $res = vm_qmp_command($vmid, $cmd);
6192
6193     my ($current, $default);
6194     foreach my $e (@$res) {
6195         $default = $e->{name} if $e->{'is-default'};
6196         $current = $e->{name} if $e->{'is-current'};
6197     }
6198
6199     # fallback to the default machine if current is not supported by qemu
6200     return $current || $default || 'pc';
6201 }
6202
6203 sub get_running_qemu_version {
6204     my ($vmid) = @_;
6205     my $cmd = { execute => 'query-version', arguments => {} };
6206     my $res = vm_qmp_command($vmid, $cmd);
6207     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6208 }
6209
6210 sub qemu_machine_feature_enabled {
6211     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6212
6213     my $current_major;
6214     my $current_minor;
6215
6216     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6217
6218         $current_major = $3;
6219         $current_minor = $4;
6220
6221     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6222
6223         $current_major = $1;
6224         $current_minor = $2;
6225     }
6226
6227     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6228
6229
6230 }
6231
6232 sub qemu_machine_pxe {
6233     my ($vmid, $conf, $machine) = @_;
6234
6235     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6236
6237     foreach my $opt (keys %$conf) {
6238         next if $opt !~ m/^net(\d+)$/;
6239         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6240         next if !$net;
6241         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6242         return $machine.".pxe" if $romfile =~ m/pxe/;
6243         last;
6244     }
6245
6246     return $machine;
6247 }
6248
6249 sub qemu_use_old_bios_files {
6250     my ($machine_type) = @_;
6251
6252     return if !$machine_type;
6253
6254     my $use_old_bios_files = undef;
6255
6256     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6257         $machine_type = $1;
6258         $use_old_bios_files = 1;
6259     } else {
6260         my $kvmver = kvm_user_version();
6261         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6262         # load new efi bios files on migration. So this hack is required to allow
6263         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6264         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6265         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6266     }
6267
6268     return ($use_old_bios_files, $machine_type);
6269 }
6270
6271 sub lspci {
6272
6273     my $devices = {};
6274
6275     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6276             my (undef, $id, $function) = @_;
6277             my $res = { id => $id, function => $function};
6278             push @{$devices->{$id}}, $res;
6279     });
6280
6281     # Entries should be sorted by functions.
6282     foreach my $id (keys %$devices) {
6283         my $dev = $devices->{$id};
6284         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6285     }
6286
6287     return $devices;
6288 }
6289
6290 sub vm_iothreads_list {
6291     my ($vmid) = @_;
6292
6293     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6294
6295     my $iothreads = {};
6296     foreach my $iothread (@$res) {
6297         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6298     }
6299
6300     return $iothreads;
6301 }
6302
6303 sub scsihw_infos {
6304     my ($conf, $drive) = @_;
6305
6306     my $maxdev = 0;
6307
6308     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6309         $maxdev = 7;
6310     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6311         $maxdev = 1;
6312     } else {
6313         $maxdev = 256;
6314     }
6315
6316     my $controller = int($drive->{index} / $maxdev);
6317     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6318
6319     return ($maxdev, $controller, $controller_prefix);
6320 }
6321
6322 sub add_hyperv_enlighments {
6323     my ($cpuFlags, $winversion, $machine_type, $kvmver, $nokvm, $bios, $gpu_passthrough) = @_;
6324
6325     return if $nokvm;
6326     return if $winversion < 6;
6327     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6328
6329     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6330
6331     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6332         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6333         push @$cpuFlags , 'hv_vapic';
6334         push @$cpuFlags , 'hv_time';
6335     } else {
6336         push @$cpuFlags , 'hv_spinlocks=0xffff';
6337     }
6338
6339     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6340         push @$cpuFlags , 'hv_reset';
6341         push @$cpuFlags , 'hv_vpindex';
6342         push @$cpuFlags , 'hv_runtime';
6343     }
6344
6345     if ($winversion >= 7) {
6346         push @$cpuFlags , 'hv_relaxed';
6347     }
6348 }
6349
6350 sub windows_version {
6351     my ($ostype) = @_;
6352
6353     return 0 if !$ostype;
6354
6355     my $winversion = 0;
6356
6357     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6358         $winversion = 5;
6359     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6360         $winversion = 6;
6361     } elsif ($ostype =~ m/^win(\d+)$/) {
6362         $winversion = $1;
6363     }
6364
6365     return $winversion;
6366 }
6367
6368 # bash completion helper
6369
6370 sub complete_backup_archives {
6371     my ($cmdname, $pname, $cvalue) = @_;
6372
6373     my $cfg = PVE::Storage::config();
6374
6375     my $storeid;
6376
6377     if ($cvalue =~ m/^([^:]+):/) {
6378         $storeid = $1;
6379     }
6380
6381     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6382
6383     my $res = [];
6384     foreach my $id (keys %$data) {
6385         foreach my $item (@{$data->{$id}}) {
6386             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6387             push @$res, $item->{volid} if defined($item->{volid});
6388         }
6389     }
6390
6391     return $res;
6392 }
6393
6394 my $complete_vmid_full = sub {
6395     my ($running) = @_;
6396
6397     my $idlist = vmstatus();
6398
6399     my $res = [];
6400
6401     foreach my $id (keys %$idlist) {
6402         my $d = $idlist->{$id};
6403         if (defined($running)) {
6404             next if $d->{template};
6405             next if $running && $d->{status} ne 'running';
6406             next if !$running && $d->{status} eq 'running';
6407         }
6408         push @$res, $id;
6409
6410     }
6411     return $res;
6412 };
6413
6414 sub complete_vmid {
6415     return &$complete_vmid_full();
6416 }
6417
6418 sub complete_vmid_stopped {
6419     return &$complete_vmid_full(0);
6420 }
6421
6422 sub complete_vmid_running {
6423     return &$complete_vmid_full(1);
6424 }
6425
6426 sub complete_storage {
6427
6428     my $cfg = PVE::Storage::config();
6429     my $ids = $cfg->{ids};
6430
6431     my $res = [];
6432     foreach my $sid (keys %$ids) {
6433         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6434         next if !$ids->{$sid}->{content}->{images};
6435         push @$res, $sid;
6436     }
6437
6438     return $res;
6439 }
6440
6441 sub nbd_stop {
6442     my ($vmid) = @_;
6443
6444     vm_mon_cmd($vmid, 'nbd-server-stop');
6445 }
6446
6447 1;