PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42
  43 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  44
  45 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  46
  47 # Note about locking: we use flock on the config file protect
  48 # against concurent actions.
  49 # Aditionaly, we have a 'lock' setting in the config file. This
  50 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  51 # allowed when such lock is set. But you can ignore this kind of
  52 # lock with the --skiplock flag.
  53
  54 cfs_register_file('/qemu-server/',
  55                   \&parse_vm_config,
  56                   \&write_vm_config);
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  72     type => 'string',
  73     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  74     description => "The drive's backing file's data format.",
  75     optional => 1,
  76 });
  77
  78 #no warnings 'redefine';
  79
  80 sub cgroups_write {
  81    my ($controller, $vmid, $option, $value) = @_;
  82
  83    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  84    PVE::ProcFSTools::write_proc_entry($path, $value);
  85
  86 }
  87
  88 my $nodename = PVE::INotify::nodename();
  89
  90 mkdir "/etc/pve/nodes/$nodename";
  91 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  92 mkdir $confdir;
  93
  94 my $var_run_tmpdir = "/var/run/qemu-server";
  95 mkdir $var_run_tmpdir;
  96
  97 my $lock_dir = "/var/lock/qemu-server";
  98 mkdir $lock_dir;
  99
 100 my $pcisysfs = "/sys/bus/pci";
 101
 102 my $cpu_vendor_list = {
 103     # Intel CPUs
 104     486 => 'GenuineIntel',
 105     pentium => 'GenuineIntel',
 106     pentium2  => 'GenuineIntel',
 107     pentium3  => 'GenuineIntel',
 108     coreduo => 'GenuineIntel',
 109     core2duo => 'GenuineIntel',
 110     Conroe  => 'GenuineIntel',
 111     Penryn  => 'GenuineIntel',
 112     Nehalem  => 'GenuineIntel',
 113     Westmere => 'GenuineIntel',
 114     SandyBridge => 'GenuineIntel',
 115     IvyBridge => 'GenuineIntel',
 116     Haswell => 'GenuineIntel',
 117     'Haswell-noTSX' => 'GenuineIntel',
 118     Broadwell => 'GenuineIntel',
 119     'Broadwell-noTSX' => 'GenuineIntel',
 120     'Skylake-Client' => 'GenuineIntel',
 121
 122     # AMD CPUs
 123     athlon => 'AuthenticAMD',
 124     phenom  => 'AuthenticAMD',
 125     Opteron_G1  => 'AuthenticAMD',
 126     Opteron_G2  => 'AuthenticAMD',
 127     Opteron_G3  => 'AuthenticAMD',
 128     Opteron_G4  => 'AuthenticAMD',
 129     Opteron_G5  => 'AuthenticAMD',
 130
 131     # generic types, use vendor from host node
 132     host => 'default',
 133     kvm32 => 'default',
 134     kvm64 => 'default',
 135     qemu32 => 'default',
 136     qemu64 => 'default',
 137 };
 138
 139 my $cpu_fmt = {
 140     cputype => {
 141         description => "Emulated CPU type.",
 142         type => 'string',
 143         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 144         default => 'kvm64',
 145         default_key => 1,
 146     },
 147     hidden => {
 148         description => "Do not identify as a KVM virtual machine.",
 149         type => 'boolean',
 150         optional => 1,
 151         default => 0
 152     },
 153 };
 154
 155 my $watchdog_fmt = {
 156     model => {
 157         default_key => 1,
 158         type => 'string',
 159         enum => [qw(i6300esb ib700)],
 160         description => "Watchdog type to emulate.",
 161         default => 'i6300esb',
 162         optional => 1,
 163     },
 164     action => {
 165         type => 'string',
 166         enum => [qw(reset shutdown poweroff pause debug none)],
 167         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 168         optional => 1,
 169     },
 170 };
 171 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 172
 173 my $confdesc = {
 174     onboot => {
 175         optional => 1,
 176         type => 'boolean',
 177         description => "Specifies whether a VM will be started during system bootup.",
 178         default => 0,
 179     },
 180     autostart => {
 181         optional => 1,
 182         type => 'boolean',
 183         description => "Automatic restart after crash (currently ignored).",
 184         default => 0,
 185     },
 186     hotplug => {
 187         optional => 1,
 188         type => 'string', format => 'pve-hotplug-features',
 189         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 190         default => 'network,disk,usb',
 191     },
 192     reboot => {
 193         optional => 1,
 194         type => 'boolean',
 195         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 196         default => 1,
 197     },
 198     lock => {
 199         optional => 1,
 200         type => 'string',
 201         description => "Lock/unlock the VM.",
 202         enum => [qw(migrate backup snapshot rollback)],
 203     },
 204     cpulimit => {
 205         optional => 1,
 206         type => 'number',
 207         description => "Limit of CPU usage.",
 208         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 209         minimum => 0,
 210         maximum => 128,
 211         default => 0,
 212     },
 213     cpuunits => {
 214         optional => 1,
 215         type => 'integer',
 216         description => "CPU weight for a VM.",
 217         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 218         minimum => 2,
 219         maximum => 262144,
 220         default => 1024,
 221     },
 222     memory => {
 223         optional => 1,
 224         type => 'integer',
 225         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 226         minimum => 16,
 227         default => 512,
 228     },
 229     balloon => {
 230         optional => 1,
 231         type => 'integer',
 232         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 233         minimum => 0,
 234     },
 235     shares => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 239         minimum => 0,
 240         maximum => 50000,
 241         default => 1000,
 242     },
 243     keyboard => {
 244         optional => 1,
 245         type => 'string',
 246         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.",
 247         enum => PVE::Tools::kvmkeymaplist(),
 248         default => 'en-us',
 249     },
 250     name => {
 251         optional => 1,
 252         type => 'string', format => 'dns-name',
 253         description => "Set a name for the VM. Only used on the configuration web interface.",
 254     },
 255     scsihw => {
 256         optional => 1,
 257         type => 'string',
 258         description => "SCSI controller model",
 259         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 260         default => 'lsi',
 261     },
 262     description => {
 263         optional => 1,
 264         type => 'string',
 265         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 266     },
 267     ostype => {
 268         optional => 1,
 269         type => 'string',
 270         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 271         description => "Specify guest operating system.",
 272         verbose_description => <<EODESC,
 273 Specify guest operating system. This is used to enable special
 274 optimization/features for specific operating systems:
 275
 276 [horizontal]
 277 other;; unspecified OS
 278 wxp;; Microsoft Windows XP
 279 w2k;; Microsoft Windows 2000
 280 w2k3;; Microsoft Windows 2003
 281 w2k8;; Microsoft Windows 2008
 282 wvista;; Microsoft Windows Vista
 283 win7;; Microsoft Windows 7
 284 win8;; Microsoft Windows 8/2012/2012r2
 285 win10;; Microsoft Windows 10/2016
 286 l24;; Linux 2.4 Kernel
 287 l26;; Linux 2.6/3.X Kernel
 288 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 289 EODESC
 290     },
 291     boot => {
 292         optional => 1,
 293         type => 'string',
 294         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 295         pattern => '[acdn]{1,4}',
 296         default => 'cdn',
 297     },
 298     bootdisk => {
 299         optional => 1,
 300         type => 'string', format => 'pve-qm-bootdisk',
 301         description => "Enable booting from specified disk.",
 302         pattern => '(ide|sata|scsi|virtio)\d+',
 303     },
 304     smp => {
 305         optional => 1,
 306         type => 'integer',
 307         description => "The number of CPUs. Please use option -sockets instead.",
 308         minimum => 1,
 309         default => 1,
 310     },
 311     sockets => {
 312         optional => 1,
 313         type => 'integer',
 314         description => "The number of CPU sockets.",
 315         minimum => 1,
 316         default => 1,
 317     },
 318     cores => {
 319         optional => 1,
 320         type => 'integer',
 321         description => "The number of cores per socket.",
 322         minimum => 1,
 323         default => 1,
 324     },
 325     numa => {
 326         optional => 1,
 327         type => 'boolean',
 328         description => "Enable/disable NUMA.",
 329         default => 0,
 330     },
 331     hugepages => {
 332         optional => 1,
 333         type => 'string',
 334         description => "Enable/disable hugepages memory.",
 335         enum => [qw(any 2 1024)],
 336     },
 337     vcpus => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "Number of hotplugged vcpus.",
 341         minimum => 1,
 342         default => 0,
 343     },
 344     acpi => {
 345         optional => 1,
 346         type => 'boolean',
 347         description => "Enable/disable ACPI.",
 348         default => 1,
 349     },
 350     agent => {
 351         optional => 1,
 352         type => 'boolean',
 353         description => "Enable/disable Qemu GuestAgent.",
 354         default => 0,
 355     },
 356     kvm => {
 357         optional => 1,
 358         type => 'boolean',
 359         description => "Enable/disable KVM hardware virtualization.",
 360         default => 1,
 361     },
 362     tdf => {
 363         optional => 1,
 364         type => 'boolean',
 365         description => "Enable/disable time drift fix.",
 366         default => 0,
 367     },
 368     localtime => {
 369         optional => 1,
 370         type => 'boolean',
 371         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 372     },
 373     freeze => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 377     },
 378     vga => {
 379         optional => 1,
 380         type => 'string',
 381         description => "Select the VGA type.",
 382         verbose_description => "Select the VGA type. If you want to use high resolution" .
 383             " modes (>= 1280x1024x16) then you should use the options " .
 384             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 385             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 386             "display sever. For win* OS you can select how many independent " .
 387             "displays you want, Linux guests can add displays them self. " .
 388             "You can also run without any graphic card, using a serial device" .
 389             " as terminal.",
 390         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 391     },
 392     watchdog => {
 393         optional => 1,
 394         type => 'string', format => 'pve-qm-watchdog',
 395         description => "Create a virtual hardware watchdog device.",
 396         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 397             " (by a guest action), the watchdog must be periodically polled " .
 398             "by an agent inside the guest or else the watchdog will reset " .
 399             "the guest (or execute the respective action specified)",
 400     },
 401     startdate => {
 402         optional => 1,
 403         type => 'string',
 404         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 405         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 406         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 407         default => 'now',
 408     },
 409     startup =>  get_standard_option('pve-startup-order'),
 410     template => {
 411         optional => 1,
 412         type => 'boolean',
 413         description => "Enable/disable Template.",
 414         default => 0,
 415     },
 416     args => {
 417         optional => 1,
 418         type => 'string',
 419         description => "Arbitrary arguments passed to kvm.",
 420         verbose_description => <<EODESCR,
 421 Arbitrary arguments passed to kvm, for example:
 422
 423 args: -no-reboot -no-hpet
 424
 425 NOTE: this option is for experts only.
 426 EODESCR
 427     },
 428     tablet => {
 429         optional => 1,
 430         type => 'boolean',
 431         default => 1,
 432         description => "Enable/disable the USB tablet device.",
 433         verbose_description => "Enable/disable the USB tablet device. This device is " .
 434             "usually needed to allow absolute mouse positioning with VNC. " .
 435             "Else the mouse runs out of sync with normal VNC clients. " .
 436             "If you're running lots of console-only guests on one host, " .
 437             "you may consider disabling this to save some context switches. " .
 438             "This is turned off by default if you use spice (-vga=qxl).",
 439     },
 440     migrate_speed => {
 441         optional => 1,
 442         type => 'integer',
 443         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 444         minimum => 0,
 445         default => 0,
 446     },
 447     migrate_downtime => {
 448         optional => 1,
 449         type => 'number',
 450         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 451         minimum => 0,
 452         default => 0.1,
 453     },
 454     cdrom => {
 455         optional => 1,
 456         type => 'string', format => 'pve-qm-ide',
 457         typetext => '<volume>',
 458         description => "This is an alias for option -ide2",
 459     },
 460     cpu => {
 461         optional => 1,
 462         description => "Emulated CPU type.",
 463         type => 'string',
 464         format => $cpu_fmt,
 465     },
 466     parent => get_standard_option('pve-snapshot-name', {
 467         optional => 1,
 468         description => "Parent snapshot name. This is used internally, and should not be modified.",
 469     }),
 470     snaptime => {
 471         optional => 1,
 472         description => "Timestamp for snapshots.",
 473         type => 'integer',
 474         minimum => 0,
 475     },
 476     vmstate => {
 477         optional => 1,
 478         type => 'string', format => 'pve-volume-id',
 479         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 480     },
 481     vmstatestorage => get_standard_option('pve-storage-id', {
 482         description => "Default storage for VM state volumes/files.",
 483         optional => 1,
 484     }),
 485     machine => {
 486         description => "Specific the Qemu machine type.",
 487         type => 'string',
 488         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 489         maxLength => 40,
 490         optional => 1,
 491     },
 492     smbios1 => {
 493         description => "Specify SMBIOS type 1 fields.",
 494         type => 'string', format => 'pve-qm-smbios1',
 495         maxLength => 256,
 496         optional => 1,
 497     },
 498     protection => {
 499         optional => 1,
 500         type => 'boolean',
 501         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 502         default => 0,
 503     },
 504     bios => {
 505         optional => 1,
 506         type => 'string',
 507         enum => [ qw(seabios ovmf) ],
 508         description => "Select BIOS implementation.",
 509         default => 'seabios',
 510     },
 511 };
 512
 513 # what about other qemu settings ?
 514 #cpu => 'string',
 515 #machine => 'string',
 516 #fda => 'file',
 517 #fdb => 'file',
 518 #mtdblock => 'file',
 519 #sd => 'file',
 520 #pflash => 'file',
 521 #snapshot => 'bool',
 522 #bootp => 'file',
 523 ##tftp => 'dir',
 524 ##smb => 'dir',
 525 #kernel => 'file',
 526 #append => 'string',
 527 #initrd => 'file',
 528 ##soundhw => 'string',
 529
 530 while (my ($k, $v) = each %$confdesc) {
 531     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 532 }
 533
 534 my $MAX_IDE_DISKS = 4;
 535 my $MAX_SCSI_DISKS = 14;
 536 my $MAX_VIRTIO_DISKS = 16;
 537 my $MAX_SATA_DISKS = 6;
 538 my $MAX_USB_DEVICES = 5;
 539 my $MAX_NETS = 32;
 540 my $MAX_UNUSED_DISKS = 8;
 541 my $MAX_HOSTPCI_DEVICES = 4;
 542 my $MAX_SERIAL_PORTS = 4;
 543 my $MAX_PARALLEL_PORTS = 3;
 544 my $MAX_NUMA = 8;
 545
 546 my $numa_fmt = {
 547     cpus => {
 548         type => "string",
 549         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 550         description => "CPUs accessing this NUMA node.",
 551         format_description => "id[-id];...",
 552     },
 553     memory => {
 554         type => "number",
 555         description => "Amount of memory this NUMA node provides.",
 556         optional => 1,
 557     },
 558     hostnodes => {
 559         type => "string",
 560         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 561         description => "Host NUMA nodes to use.",
 562         format_description => "id[-id];...",
 563         optional => 1,
 564     },
 565     policy => {
 566         type => 'string',
 567         enum => [qw(preferred bind interleave)],
 568         description => "NUMA allocation policy.",
 569         optional => 1,
 570     },
 571 };
 572 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 573 my $numadesc = {
 574     optional => 1,
 575     type => 'string', format => $numa_fmt,
 576     description => "NUMA topology.",
 577 };
 578 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 579
 580 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 581     $confdesc->{"numa$i"} = $numadesc;
 582 }
 583
 584 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 585                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 586                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 587 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 588
 589 my $net_fmt_bridge_descr = <<__EOD__;
 590 Bridge to attach the network device to. The Proxmox VE standard bridge
 591 is called 'vmbr0'.
 592
 593 If you do not specify a bridge, we create a kvm user (NATed) network
 594 device, which provides DHCP and DNS services. The following addresses
 595 are used:
 596
 597  10.0.2.2   Gateway
 598  10.0.2.3   DNS Server
 599  10.0.2.4   SMB Server
 600
 601 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 602 __EOD__
 603
 604 my $net_fmt = {
 605     macaddr => {
 606         type => 'string',
 607         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 608         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 609         format_description => "XX:XX:XX:XX:XX:XX",
 610         optional => 1,
 611     },
 612     model => {
 613         type => 'string',
 614         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 615         enum => $nic_model_list,
 616         default_key => 1,
 617     },
 618     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 619     bridge => {
 620         type => 'string',
 621         description => $net_fmt_bridge_descr,
 622         format_description => 'bridge',
 623         optional => 1,
 624     },
 625     queues => {
 626         type => 'integer',
 627         minimum => 0, maximum => 16,
 628         description => 'Number of packet queues to be used on the device.',
 629         optional => 1,
 630     },
 631     rate => {
 632         type => 'number',
 633         minimum => 0,
 634         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 635         optional => 1,
 636     },
 637     tag => {
 638         type => 'integer',
 639         minimum => 1, maximum => 4094,
 640         description => 'VLAN tag to apply to packets on this interface.',
 641         optional => 1,
 642     },
 643     trunks => {
 644         type => 'string',
 645         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 646         description => 'VLAN trunks to pass through this interface.',
 647         format_description => 'vlanid[;vlanid...]',
 648         optional => 1,
 649     },
 650     firewall => {
 651         type => 'boolean',
 652         description => 'Whether this interface should be protected by the firewall.',
 653         optional => 1,
 654     },
 655     link_down => {
 656         type => 'boolean',
 657         description => 'Whether this interface should be disconnected (like pulling the plug).',
 658         optional => 1,
 659     },
 660 };
 661
 662 my $netdesc = {
 663     optional => 1,
 664     type => 'string', format => $net_fmt,
 665     description => "Specify network devices.",
 666 };
 667
 668 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 669
 670 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 671     $confdesc->{"net$i"} = $netdesc;
 672 }
 673
 674 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 675 sub verify_volume_id_or_qm_path {
 676     my ($volid, $noerr) = @_;
 677
 678     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 679         return $volid;
 680     }
 681
 682     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 683     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 684     if ($@) {
 685         return undef if $noerr;
 686         die $@;
 687     }
 688     return $volid;
 689 }
 690
 691 my $drivename_hash;
 692
 693 my %drivedesc_base = (
 694     volume => { alias => 'file' },
 695     file => {
 696         type => 'string',
 697         format => 'pve-volume-id-or-qm-path',
 698         default_key => 1,
 699         format_description => 'volume',
 700         description => "The drive's backing volume.",
 701     },
 702     media => {
 703         type => 'string',
 704         enum => [qw(cdrom disk)],
 705         description => "The drive's media type.",
 706         default => 'disk',
 707         optional => 1
 708     },
 709     cyls => {
 710         type => 'integer',
 711         description => "Force the drive's physical geometry to have a specific cylinder count.",
 712         optional => 1
 713     },
 714     heads => {
 715         type => 'integer',
 716         description => "Force the drive's physical geometry to have a specific head count.",
 717         optional => 1
 718     },
 719     secs => {
 720         type => 'integer',
 721         description => "Force the drive's physical geometry to have a specific sector count.",
 722         optional => 1
 723     },
 724     trans => {
 725         type => 'string',
 726         enum => [qw(none lba auto)],
 727         description => "Force disk geometry bios translation mode.",
 728         optional => 1,
 729     },
 730     snapshot => {
 731         type => 'boolean',
 732         description => "Whether the drive should be included when making snapshots.",
 733         optional => 1,
 734     },
 735     cache => {
 736         type => 'string',
 737         enum => [qw(none writethrough writeback unsafe directsync)],
 738         description => "The drive's cache mode",
 739         optional => 1,
 740     },
 741     format => get_standard_option('pve-qm-image-format'),
 742     size => {
 743         type => 'string',
 744         format => 'disk-size',
 745         format_description => 'DiskSize',
 746         description => "Disk size. This is purely informational and has no effect.",
 747         optional => 1,
 748     },
 749     backup => {
 750         type => 'boolean',
 751         description => "Whether the drive should be included when making backups.",
 752         optional => 1,
 753     },
 754     replicate => {
 755         type => 'boolean',
 756         description => 'Whether the drive should considered for replication jobs.',
 757         optional => 1,
 758         default => 1,
 759     },
 760     rerror => {
 761         type => 'string',
 762         enum => [qw(ignore report stop)],
 763         description => 'Read error action.',
 764         optional => 1,
 765     },
 766     werror => {
 767         type => 'string',
 768         enum => [qw(enospc ignore report stop)],
 769         description => 'Write error action.',
 770         optional => 1,
 771     },
 772     aio => {
 773         type => 'string',
 774         enum => [qw(native threads)],
 775         description => 'AIO type to use.',
 776         optional => 1,
 777     },
 778     discard => {
 779         type => 'string',
 780         enum => [qw(ignore on)],
 781         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 782         optional => 1,
 783     },
 784     detect_zeroes => {
 785         type => 'boolean',
 786         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 787         optional => 1,
 788     },
 789     serial => {
 790         type => 'string',
 791         format => 'urlencoded',
 792         format_description => 'serial',
 793         maxLength => 20*3, # *3 since it's %xx url enoded
 794         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 795         optional => 1,
 796     }
 797 );
 798
 799 my %iothread_fmt = ( iothread => {
 800         type => 'boolean',
 801         description => "Whether to use iothreads for this drive",
 802         optional => 1,
 803 });
 804
 805 my %model_fmt = (
 806     model => {
 807         type => 'string',
 808         format => 'urlencoded',
 809         format_description => 'model',
 810         maxLength => 40*3, # *3 since it's %xx url enoded
 811         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 812         optional => 1,
 813     },
 814 );
 815
 816 my %queues_fmt = (
 817     queues => {
 818         type => 'integer',
 819         description => "Number of queues.",
 820         minimum => 2,
 821         optional => 1
 822     }
 823 );
 824
 825 my %scsiblock_fmt = (
 826     scsiblock => {
 827         type => 'boolean',
 828         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 829         optional => 1,
 830         default => 0,
 831     },
 832 );
 833
 834 my $add_throttle_desc = sub {
 835     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 836     my $d = {
 837         type => $type,
 838         format_description => $unit,
 839         description => "Maximum $what in $longunit.",
 840         optional => 1,
 841     };
 842     $d->{minimum} = $minimum if defined($minimum);
 843     $drivedesc_base{$key} = $d;
 844 };
 845 # throughput: (leaky bucket)
 846 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 847 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 848 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 849 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 850 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 851 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 852 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 853 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 854 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 855
 856 # pools: (pool of IO before throttling starts taking effect)
 857 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 858 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 859 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 860 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 861 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 862 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 863
 864 # burst lengths
 865 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 866 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 867 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 868 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 869 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 870 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 871
 872 # legacy support
 873 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
 874 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
 875 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
 876 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
 877
 878 my $ide_fmt = {
 879     %drivedesc_base,
 880     %model_fmt,
 881 };
 882 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 883
 884 my $idedesc = {
 885     optional => 1,
 886     type => 'string', format => $ide_fmt,
 887     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 888 };
 889 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 890
 891 my $scsi_fmt = {
 892     %drivedesc_base,
 893     %iothread_fmt,
 894     %queues_fmt,
 895     %scsiblock_fmt,
 896 };
 897 my $scsidesc = {
 898     optional => 1,
 899     type => 'string', format => $scsi_fmt,
 900     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 901 };
 902 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 903
 904 my $sata_fmt = {
 905     %drivedesc_base,
 906 };
 907 my $satadesc = {
 908     optional => 1,
 909     type => 'string', format => $sata_fmt,
 910     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 911 };
 912 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 913
 914 my $virtio_fmt = {
 915     %drivedesc_base,
 916     %iothread_fmt,
 917 };
 918 my $virtiodesc = {
 919     optional => 1,
 920     type => 'string', format => $virtio_fmt,
 921     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 922 };
 923 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 924
 925 my $alldrive_fmt = {
 926     %drivedesc_base,
 927     %iothread_fmt,
 928     %model_fmt,
 929     %queues_fmt,
 930     %scsiblock_fmt,
 931 };
 932
 933 my $efidisk_fmt = {
 934     volume => { alias => 'file' },
 935     file => {
 936         type => 'string',
 937         format => 'pve-volume-id-or-qm-path',
 938         default_key => 1,
 939         format_description => 'volume',
 940         description => "The drive's backing volume.",
 941     },
 942     format => get_standard_option('pve-qm-image-format'),
 943     size => {
 944         type => 'string',
 945         format => 'disk-size',
 946         format_description => 'DiskSize',
 947         description => "Disk size. This is purely informational and has no effect.",
 948         optional => 1,
 949     },
 950 };
 951
 952 my $efidisk_desc = {
 953     optional => 1,
 954     type => 'string', format => $efidisk_fmt,
 955     description => "Configure a Disk for storing EFI vars",
 956 };
 957
 958 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 959
 960 my $usb_fmt = {
 961     host => {
 962         default_key => 1,
 963         type => 'string', format => 'pve-qm-usb-device',
 964         format_description => 'HOSTUSBDEVICE|spice',
 965         description => <<EODESCR,
 966 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 967
 968  'bus-port(.port)*' (decimal numbers) or
 969  'vendor_id:product_id' (hexadeciaml numbers) or
 970  'spice'
 971
 972 You can use the 'lsusb -t' command to list existing usb devices.
 973
 974 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 975
 976 The value 'spice' can be used to add a usb redirection devices for spice.
 977 EODESCR
 978     },
 979     usb3 => {
 980         optional => 1,
 981         type => 'boolean',
 982         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 983         default => 0,
 984     },
 985 };
 986
 987 my $usbdesc = {
 988     optional => 1,
 989     type => 'string', format => $usb_fmt,
 990     description => "Configure an USB device (n is 0 to 4).",
 991 };
 992 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
 993
 994 # NOTE: the match-groups of this regex are used in parse_hostpci
 995 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
 996 my $hostpci_fmt = {
 997     host => {
 998         default_key => 1,
 999         type => 'string',
1000         pattern => qr/$PCIRE(;$PCIRE)*/,
1001         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1002         description => <<EODESCR,
1003 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1004 of PCI virtual functions of the host. HOSTPCIID syntax is:
1005
1006 'bus:dev.func' (hexadecimal numbers)
1007
1008 You can us the 'lspci' command to list existing PCI devices.
1009 EODESCR
1010     },
1011     rombar => {
1012         type => 'boolean',
1013         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1014         optional => 1,
1015         default => 1,
1016     },
1017     romfile => {
1018         type => 'string',
1019         pattern => '[^,;]+',
1020         format_description => 'string',
1021         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1022         optional => 1,
1023     },
1024     pcie => {
1025         type => 'boolean',
1026         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1027         optional => 1,
1028         default => 0,
1029     },
1030     'x-vga' => {
1031         type => 'boolean',
1032         description =>  "Enable vfio-vga device support.",
1033         optional => 1,
1034         default => 0,
1035     },
1036 };
1037 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1038
1039 my $hostpcidesc = {
1040         optional => 1,
1041         type => 'string', format => 'pve-qm-hostpci',
1042         description => "Map host PCI devices into guest.",
1043         verbose_description =>  <<EODESCR,
1044 Map host PCI devices into guest.
1045
1046 NOTE: This option allows direct access to host hardware. So it is no longer
1047 possible to migrate such machines - use with special care.
1048
1049 CAUTION: Experimental! User reported problems with this option.
1050 EODESCR
1051 };
1052 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1053
1054 my $serialdesc = {
1055         optional => 1,
1056         type => 'string',
1057         pattern => '(/dev/.+|socket)',
1058         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1059         verbose_description =>  <<EODESCR,
1060 Create a serial device inside the VM (n is 0 to 3), and pass through a
1061 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1062 host side (use 'qm terminal' to open a terminal connection).
1063
1064 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1065
1066 CAUTION: Experimental! User reported problems with this option.
1067 EODESCR
1068 };
1069
1070 my $paralleldesc= {
1071         optional => 1,
1072         type => 'string',
1073         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1074         description =>  "Map host parallel devices (n is 0 to 2).",
1075         verbose_description =>  <<EODESCR,
1076 Map host parallel devices (n is 0 to 2).
1077
1078 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1079
1080 CAUTION: Experimental! User reported problems with this option.
1081 EODESCR
1082 };
1083
1084 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1085     $confdesc->{"parallel$i"} = $paralleldesc;
1086 }
1087
1088 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1089     $confdesc->{"serial$i"} = $serialdesc;
1090 }
1091
1092 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1093     $confdesc->{"hostpci$i"} = $hostpcidesc;
1094 }
1095
1096 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1097     $drivename_hash->{"ide$i"} = 1;
1098     $confdesc->{"ide$i"} = $idedesc;
1099 }
1100
1101 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1102     $drivename_hash->{"sata$i"} = 1;
1103     $confdesc->{"sata$i"} = $satadesc;
1104 }
1105
1106 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1107     $drivename_hash->{"scsi$i"} = 1;
1108     $confdesc->{"scsi$i"} = $scsidesc ;
1109 }
1110
1111 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1112     $drivename_hash->{"virtio$i"} = 1;
1113     $confdesc->{"virtio$i"} = $virtiodesc;
1114 }
1115
1116 $drivename_hash->{efidisk0} = 1;
1117 $confdesc->{efidisk0} = $efidisk_desc;
1118
1119 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1120     $confdesc->{"usb$i"} = $usbdesc;
1121 }
1122
1123 my $unuseddesc = {
1124     optional => 1,
1125     type => 'string', format => 'pve-volume-id',
1126     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1127 };
1128
1129 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1130     $confdesc->{"unused$i"} = $unuseddesc;
1131 }
1132
1133 my $kvm_api_version = 0;
1134
1135 sub kvm_version {
1136
1137     return $kvm_api_version if $kvm_api_version;
1138
1139     my $fh = IO::File->new("</dev/kvm") ||
1140         return 0;
1141
1142     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1143         $kvm_api_version = $v;
1144     }
1145
1146     $fh->close();
1147
1148     return  $kvm_api_version;
1149 }
1150
1151 my $kvm_user_version;
1152
1153 sub kvm_user_version {
1154
1155     return $kvm_user_version if $kvm_user_version;
1156
1157     $kvm_user_version = 'unknown';
1158
1159     my $code = sub {
1160         my $line = shift;
1161         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1162             $kvm_user_version = $2;
1163         }
1164     };
1165
1166     eval { run_command("kvm -version", outfunc => $code); };
1167     warn $@ if $@;
1168
1169     return $kvm_user_version;
1170
1171 }
1172
1173 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1174
1175 sub valid_drive_names {
1176     # order is important - used to autoselect boot disk
1177     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1178             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1179             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1180             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1181             'efidisk0');
1182 }
1183
1184 sub is_valid_drivename {
1185     my $dev = shift;
1186
1187     return defined($drivename_hash->{$dev});
1188 }
1189
1190 sub option_exists {
1191     my $key = shift;
1192     return defined($confdesc->{$key});
1193 }
1194
1195 sub nic_models {
1196     return $nic_model_list;
1197 }
1198
1199 sub os_list_description {
1200
1201     return {
1202         other => 'Other',
1203         wxp => 'Windows XP',
1204         w2k => 'Windows 2000',
1205         w2k3 =>, 'Windows 2003',
1206         w2k8 => 'Windows 2008',
1207         wvista => 'Windows Vista',
1208         win7 => 'Windows 7',
1209         win8 => 'Windows 8/2012',
1210         win10 => 'Windows 10/2016',
1211         l24 => 'Linux 2.4',
1212         l26 => 'Linux 2.6',
1213     };
1214 }
1215
1216 my $cdrom_path;
1217
1218 sub get_cdrom_path {
1219
1220     return  $cdrom_path if $cdrom_path;
1221
1222     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1223     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1224     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1225 }
1226
1227 sub get_iso_path {
1228     my ($storecfg, $vmid, $cdrom) = @_;
1229
1230     if ($cdrom eq 'cdrom') {
1231         return get_cdrom_path();
1232     } elsif ($cdrom eq 'none') {
1233         return '';
1234     } elsif ($cdrom =~ m|^/|) {
1235         return $cdrom;
1236     } else {
1237         return PVE::Storage::path($storecfg, $cdrom);
1238     }
1239 }
1240
1241 # try to convert old style file names to volume IDs
1242 sub filename_to_volume_id {
1243     my ($vmid, $file, $media) = @_;
1244
1245     if (!($file eq 'none' || $file eq 'cdrom' ||
1246           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1247
1248         return undef if $file =~ m|/|;
1249
1250         if ($media && $media eq 'cdrom') {
1251             $file = "local:iso/$file";
1252         } else {
1253             $file = "local:$vmid/$file";
1254         }
1255     }
1256
1257     return $file;
1258 }
1259
1260 sub verify_media_type {
1261     my ($opt, $vtype, $media) = @_;
1262
1263     return if !$media;
1264
1265     my $etype;
1266     if ($media eq 'disk') {
1267         $etype = 'images';
1268     } elsif ($media eq 'cdrom') {
1269         $etype = 'iso';
1270     } else {
1271         die "internal error";
1272     }
1273
1274     return if ($vtype eq $etype);
1275
1276     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1277 }
1278
1279 sub cleanup_drive_path {
1280     my ($opt, $storecfg, $drive) = @_;
1281
1282     # try to convert filesystem paths to volume IDs
1283
1284     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1285         ($drive->{file} !~ m|^/dev/.+|) &&
1286         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1287         ($drive->{file} !~ m/^\d+$/)) {
1288         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1289         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1290         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1291         verify_media_type($opt, $vtype, $drive->{media});
1292         $drive->{file} = $volid;
1293     }
1294
1295     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1296 }
1297
1298 sub parse_hotplug_features {
1299     my ($data) = @_;
1300
1301     my $res = {};
1302
1303     return $res if $data eq '0';
1304
1305     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1306
1307     foreach my $feature (PVE::Tools::split_list($data)) {
1308         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1309             $res->{$1} = 1;
1310         } else {
1311             die "invalid hotplug feature '$feature'\n";
1312         }
1313     }
1314     return $res;
1315 }
1316
1317 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1318 sub pve_verify_hotplug_features {
1319     my ($value, $noerr) = @_;
1320
1321     return $value if parse_hotplug_features($value);
1322
1323     return undef if $noerr;
1324
1325     die "unable to parse hotplug option\n";
1326 }
1327
1328 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1329 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1330 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1331 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1332 #        [,iothread=on][,serial=serial][,model=model]
1333
1334 sub parse_drive {
1335     my ($key, $data) = @_;
1336
1337     my ($interface, $index);
1338
1339     if ($key =~ m/^([^\d]+)(\d+)$/) {
1340         $interface = $1;
1341         $index = $2;
1342     } else {
1343         return undef;
1344     }
1345
1346     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1347                                      : $confdesc->{$key}->{format};
1348     if (!$desc) {
1349         warn "invalid drive key: $key\n";
1350         return undef;
1351     }
1352     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1353     return undef if !$res;
1354     $res->{interface} = $interface;
1355     $res->{index} = $index;
1356
1357     my $error = 0;
1358     foreach my $opt (qw(bps bps_rd bps_wr)) {
1359         if (my $bps = defined(delete $res->{$opt})) {
1360             if (defined($res->{"m$opt"})) {
1361                 warn "both $opt and m$opt specified\n";
1362                 ++$error;
1363                 next;
1364             }
1365             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1366         }
1367     }
1368
1369     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1370     for my $requirement (
1371         [mbps_max => 'mbps'],
1372         [mbps_rd_max => 'mbps_rd'],
1373         [mbps_wr_max => 'mbps_wr'],
1374         [miops_max => 'miops'],
1375         [miops_rd_max => 'miops_rd'],
1376         [miops_wr_max => 'miops_wr'],
1377         [bps_max_length => 'mbps_max'],
1378         [bps_rd_max_length => 'mbps_rd_max'],
1379         [bps_wr_max_length => 'mbps_wr_max'],
1380         [iops_max_length => 'iops_max'],
1381         [iops_rd_max_length => 'iops_rd_max'],
1382         [iops_wr_max_length => 'iops_wr_max']) {
1383         my ($option, $requires) = @$requirement;
1384         if ($res->{$option} && !$res->{$requires}) {
1385             warn "$option requires $requires\n";
1386             ++$error;
1387         }
1388     }
1389
1390     return undef if $error;
1391
1392     return undef if $res->{mbps_rd} && $res->{mbps};
1393     return undef if $res->{mbps_wr} && $res->{mbps};
1394     return undef if $res->{iops_rd} && $res->{iops};
1395     return undef if $res->{iops_wr} && $res->{iops};
1396
1397     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1398         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1399         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1400         return undef if $res->{interface} eq 'virtio';
1401     }
1402
1403     if (my $size = $res->{size}) {
1404         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1405     }
1406
1407     return $res;
1408 }
1409
1410 sub print_drive {
1411     my ($vmid, $drive) = @_;
1412     my $data = { %$drive };
1413     delete $data->{$_} for qw(index interface);
1414     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1415 }
1416
1417 sub scsi_inquiry {
1418     my($fh, $noerr) = @_;
1419
1420     my $SG_IO = 0x2285;
1421     my $SG_GET_VERSION_NUM = 0x2282;
1422
1423     my $versionbuf = "\x00" x 8;
1424     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1425     if (!$ret) {
1426         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1427         return undef;
1428     }
1429     my $version = unpack("I", $versionbuf);
1430     if ($version < 30000) {
1431         die "scsi generic interface too old\n"  if !$noerr;
1432         return undef;
1433     }
1434
1435     my $buf = "\x00" x 36;
1436     my $sensebuf = "\x00" x 8;
1437     my $cmd = pack("C x3 C x1", 0x12, 36);
1438
1439     # see /usr/include/scsi/sg.h
1440     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1441
1442     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1443                       length($sensebuf), 0, length($buf), $buf,
1444                       $cmd, $sensebuf, 6000);
1445
1446     $ret = ioctl($fh, $SG_IO, $packet);
1447     if (!$ret) {
1448         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1449         return undef;
1450     }
1451
1452     my @res = unpack($sg_io_hdr_t, $packet);
1453     if ($res[17] || $res[18]) {
1454         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1455         return undef;
1456     }
1457
1458     my $res = {};
1459     (my $byte0, my $byte1, $res->{vendor},
1460      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1461
1462     $res->{removable} = $byte1 & 128 ? 1 : 0;
1463     $res->{type} = $byte0 & 31;
1464
1465     return $res;
1466 }
1467
1468 sub path_is_scsi {
1469     my ($path) = @_;
1470
1471     my $fh = IO::File->new("+<$path") || return undef;
1472     my $res = scsi_inquiry($fh, 1);
1473     close($fh);
1474
1475     return $res;
1476 }
1477
1478 sub machine_type_is_q35 {
1479     my ($conf) = @_;
1480
1481     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1482 }
1483
1484 sub print_tabletdevice_full {
1485     my ($conf) = @_;
1486
1487     my $q35 = machine_type_is_q35($conf);
1488
1489     # we use uhci for old VMs because tablet driver was buggy in older qemu
1490     my $usbbus = $q35 ? "ehci" : "uhci";
1491
1492     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1493 }
1494
1495 sub print_drivedevice_full {
1496     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1497
1498     my $device = '';
1499     my $maxdev = 0;
1500
1501     if ($drive->{interface} eq 'virtio') {
1502         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1503         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1504         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1505     } elsif ($drive->{interface} eq 'scsi') {
1506
1507         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1508         my $unit = $drive->{index} % $maxdev;
1509         my $devicetype = 'hd';
1510         my $path = '';
1511         if (drive_is_cdrom($drive)) {
1512             $devicetype = 'cd';
1513         } else {
1514             if ($drive->{file} =~ m|^/|) {
1515                 $path = $drive->{file};
1516                 if (my $info = path_is_scsi($path)) {
1517                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1518                         $devicetype = 'block';
1519                     } elsif ($info->{type} == 1) { # tape
1520                         $devicetype = 'generic';
1521                     }
1522                 }
1523             } else {
1524                  $path = PVE::Storage::path($storecfg, $drive->{file});
1525             }
1526
1527             if($path =~ m/^iscsi\:\/\//){
1528                 $devicetype = 'generic';
1529             }
1530         }
1531
1532         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1533             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1534         } else {
1535             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1536         }
1537
1538     } elsif ($drive->{interface} eq 'ide'){
1539         $maxdev = 2;
1540         my $controller = int($drive->{index} / $maxdev);
1541         my $unit = $drive->{index} % $maxdev;
1542         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1543
1544         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1545         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1546             $model = URI::Escape::uri_unescape($model);
1547             $device .= ",model=$model";
1548         }
1549     } elsif ($drive->{interface} eq 'sata'){
1550         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1551         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1552         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1553     } elsif ($drive->{interface} eq 'usb') {
1554         die "implement me";
1555         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1556     } else {
1557         die "unsupported interface type";
1558     }
1559
1560     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1561
1562     return $device;
1563 }
1564
1565 sub get_initiator_name {
1566     my $initiator;
1567
1568     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1569     while (defined(my $line = <$fh>)) {
1570         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1571         $initiator = $1;
1572         last;
1573     }
1574     $fh->close();
1575
1576     return $initiator;
1577 }
1578
1579 sub print_drive_full {
1580     my ($storecfg, $vmid, $drive) = @_;
1581
1582     my $path;
1583     my $volid = $drive->{file};
1584     my $format;
1585
1586     if (drive_is_cdrom($drive)) {
1587         $path = get_iso_path($storecfg, $vmid, $volid);
1588     } else {
1589         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1590         if ($storeid) {
1591             $path = PVE::Storage::path($storecfg, $volid);
1592             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1593             $format = qemu_img_format($scfg, $volname);
1594         } else {
1595             $path = $volid;
1596             $format = "raw";
1597         }
1598    }
1599
1600     my $opts = '';
1601     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard);
1602     foreach my $o (@qemu_drive_options) {
1603         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1604     }
1605     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1606         my ($dir, $qmpname) = @$type;
1607         if (my $v = $drive->{"mbps$dir"}) {
1608             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1609         }
1610         if (my $v = $drive->{"mbps${dir}_max"}) {
1611             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1612         }
1613         if (my $v = $drive->{"bps${dir}_max_length"}) {
1614             $opts .= ",throttling.bps$qmpname-max-length=$v";
1615         }
1616         if (my $v = $drive->{"iops${dir}"}) {
1617             $opts .= ",throttling.iops$qmpname=$v";
1618         }
1619         if (my $v = $drive->{"iops${dir}_max"}) {
1620             $opts .= ",throttling.iops$qmpname-max=$v";
1621         }
1622         if (my $v = $drive->{"iops${dir}_max_length"}) {
1623             $opts .= ",throttling.iops$qmpname-max-length=$v";
1624         }
1625     }
1626
1627     if (my $serial = $drive->{serial}) {
1628         $serial = URI::Escape::uri_unescape($serial);
1629         $opts .= ",serial=$serial";
1630     }
1631
1632     $opts .= ",format=$format" if $format && !$drive->{format};
1633
1634     my $cache_direct = 0;
1635
1636     if (my $cache = $drive->{cache}) {
1637         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1638     } elsif (!drive_is_cdrom($drive)) {
1639         $opts .= ",cache=none";
1640         $cache_direct = 1;
1641     }
1642
1643     # aio native works only with O_DIRECT
1644     if (!$drive->{aio}) {
1645         if($cache_direct) {
1646             $opts .= ",aio=native";
1647         } else {
1648             $opts .= ",aio=threads";
1649         }
1650     }
1651
1652     if (!drive_is_cdrom($drive)) {
1653         my $detectzeroes;
1654         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1655             $detectzeroes = 'off';
1656         } elsif ($drive->{discard}) {
1657             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1658         } else {
1659             # This used to be our default with discard not being specified:
1660             $detectzeroes = 'on';
1661         }
1662         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1663     }
1664
1665     my $pathinfo = $path ? "file=$path," : '';
1666
1667     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1668 }
1669
1670 sub print_netdevice_full {
1671     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1672
1673     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1674
1675     my $device = $net->{model};
1676     if ($net->{model} eq 'virtio') {
1677          $device = 'virtio-net-pci';
1678      };
1679
1680     my $pciaddr = print_pci_addr("$netid", $bridges);
1681     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1682     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1683         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1684         my $vectors = $net->{queues} * 2 + 2;
1685         $tmpstr .= ",vectors=$vectors,mq=on";
1686     }
1687     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1688
1689     if ($use_old_bios_files) {
1690         my $romfile;
1691         if ($device eq 'virtio-net-pci') {
1692             $romfile = 'pxe-virtio.rom';
1693         } elsif ($device eq 'e1000') {
1694             $romfile = 'pxe-e1000.rom';
1695         } elsif ($device eq 'ne2k') {
1696             $romfile = 'pxe-ne2k_pci.rom';
1697         } elsif ($device eq 'pcnet') {
1698             $romfile = 'pxe-pcnet.rom';
1699         } elsif ($device eq 'rtl8139') {
1700             $romfile = 'pxe-rtl8139.rom';
1701         }
1702         $tmpstr .= ",romfile=$romfile" if $romfile;
1703     }
1704
1705     return $tmpstr;
1706 }
1707
1708 sub print_netdev_full {
1709     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1710
1711     my $i = '';
1712     if ($netid =~ m/^net(\d+)$/) {
1713         $i = int($1);
1714     }
1715
1716     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1717
1718     my $ifname = "tap${vmid}i$i";
1719
1720     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1721     die "interface name '$ifname' is too long (max 15 character)\n"
1722         if length($ifname) >= 16;
1723
1724     my $vhostparam = '';
1725     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1726
1727     my $vmname = $conf->{name} || "vm$vmid";
1728
1729     my $netdev = "";
1730     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1731
1732     if ($net->{bridge}) {
1733         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1734     } else {
1735         $netdev = "type=user,id=$netid,hostname=$vmname";
1736     }
1737
1738     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1739
1740     return $netdev;
1741 }
1742
1743
1744 sub print_cpu_device {
1745     my ($conf, $id) = @_;
1746
1747     my $kvm = $conf->{kvm} // 1;
1748     my $cpu = $kvm ? "kvm64" : "qemu64";
1749     if (my $cputype = $conf->{cpu}) {
1750         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1751             or die "Cannot parse cpu description: $cputype\n";
1752         $cpu = $cpuconf->{cputype};
1753     }
1754
1755     my $cores = $conf->{cores} || 1;
1756
1757     my $current_core = ($id - 1) % $cores;
1758     my $current_socket = int(($id - 1 - $current_core)/$cores);
1759
1760     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1761 }
1762
1763 sub drive_is_cdrom {
1764     my ($drive) = @_;
1765
1766     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1767
1768 }
1769
1770 sub parse_number_sets {
1771     my ($set) = @_;
1772     my $res = [];
1773     foreach my $part (split(/;/, $set)) {
1774         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1775             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1776             push @$res, [ $1, $2 ];
1777         } else {
1778             die "invalid range: $part\n";
1779         }
1780     }
1781     return $res;
1782 }
1783
1784 sub parse_numa {
1785     my ($data) = @_;
1786
1787     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1788     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1789     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1790     return $res;
1791 }
1792
1793 sub parse_hostpci {
1794     my ($value) = @_;
1795
1796     return undef if !$value;
1797
1798     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1799
1800     my @idlist = split(/;/, $res->{host});
1801     delete $res->{host};
1802     foreach my $id (@idlist) {
1803         if ($id =~ /^$PCIRE$/) {
1804             if (defined($2)) {
1805                 push @{$res->{pciid}}, { id => $1, function => $2 };
1806             } else {
1807                 my $pcidevices = lspci($1);
1808                 $res->{pciid} = $pcidevices->{$1};
1809             }
1810         } else {
1811             # should have been caught by parse_property_string already
1812             die "failed to parse PCI id: $id\n";
1813         }
1814     }
1815     return $res;
1816 }
1817
1818 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1819 sub parse_net {
1820     my ($data) = @_;
1821
1822     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1823     if ($@) {
1824         warn $@;
1825         return undef;
1826     }
1827     if (!defined($res->{macaddr})) {
1828         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1829         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1830     }
1831     return $res;
1832 }
1833
1834 sub print_net {
1835     my $net = shift;
1836
1837     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1838 }
1839
1840 sub add_random_macs {
1841     my ($settings) = @_;
1842
1843     foreach my $opt (keys %$settings) {
1844         next if $opt !~ m/^net(\d+)$/;
1845         my $net = parse_net($settings->{$opt});
1846         next if !$net;
1847         $settings->{$opt} = print_net($net);
1848     }
1849 }
1850
1851 sub vm_is_volid_owner {
1852     my ($storecfg, $vmid, $volid) = @_;
1853
1854     if ($volid !~  m|^/|) {
1855         my ($path, $owner);
1856         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1857         if ($owner && ($owner == $vmid)) {
1858             return 1;
1859         }
1860     }
1861
1862     return undef;
1863 }
1864
1865 sub split_flagged_list {
1866     my $text = shift || '';
1867     $text =~ s/[,;]/ /g;
1868     $text =~ s/^\s+//;
1869     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1870 }
1871
1872 sub join_flagged_list {
1873     my ($how, $lst) = @_;
1874     join $how, map { $lst->{$_} . $_ } keys %$lst;
1875 }
1876
1877 sub vmconfig_delete_pending_option {
1878     my ($conf, $key, $force) = @_;
1879
1880     delete $conf->{pending}->{$key};
1881     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1882     $pending_delete_hash->{$key} = $force ? '!' : '';
1883     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1884 }
1885
1886 sub vmconfig_undelete_pending_option {
1887     my ($conf, $key) = @_;
1888
1889     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1890     delete $pending_delete_hash->{$key};
1891
1892     if (%$pending_delete_hash) {
1893         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1894     } else {
1895         delete $conf->{pending}->{delete};
1896     }
1897 }
1898
1899 sub vmconfig_register_unused_drive {
1900     my ($storecfg, $vmid, $conf, $drive) = @_;
1901
1902     if (!drive_is_cdrom($drive)) {
1903         my $volid = $drive->{file};
1904         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1905             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1906         }
1907     }
1908 }
1909
1910 sub vmconfig_cleanup_pending {
1911     my ($conf) = @_;
1912
1913     # remove pending changes when nothing changed
1914     my $changes;
1915     foreach my $opt (keys %{$conf->{pending}}) {
1916         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1917             $changes = 1;
1918             delete $conf->{pending}->{$opt};
1919         }
1920     }
1921
1922     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1923     my $pending_delete_hash = {};
1924     while (my ($opt, $force) = each %$current_delete_hash) {
1925         if (defined($conf->{$opt})) {
1926             $pending_delete_hash->{$opt} = $force;
1927         } else {
1928             $changes = 1;
1929         }
1930     }
1931
1932     if (%$pending_delete_hash) {
1933         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1934     } else {
1935         delete $conf->{pending}->{delete};
1936     }
1937
1938     return $changes;
1939 }
1940
1941 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1942 my $smbios1_fmt = {
1943     uuid => {
1944         type => 'string',
1945         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1946         format_description => 'UUID',
1947         description => "Set SMBIOS1 UUID.",
1948         optional => 1,
1949     },
1950     version => {
1951         type => 'string',
1952         pattern => '\S+',
1953         format_description => 'string',
1954         description => "Set SMBIOS1 version.",
1955         optional => 1,
1956     },
1957     serial => {
1958         type => 'string',
1959         pattern => '\S+',
1960         format_description => 'string',
1961         description => "Set SMBIOS1 serial number.",
1962         optional => 1,
1963     },
1964     manufacturer => {
1965         type => 'string',
1966         pattern => '\S+',
1967         format_description => 'string',
1968         description => "Set SMBIOS1 manufacturer.",
1969         optional => 1,
1970     },
1971     product => {
1972         type => 'string',
1973         pattern => '\S+',
1974         format_description => 'string',
1975         description => "Set SMBIOS1 product ID.",
1976         optional => 1,
1977     },
1978     sku => {
1979         type => 'string',
1980         pattern => '\S+',
1981         format_description => 'string',
1982         description => "Set SMBIOS1 SKU string.",
1983         optional => 1,
1984     },
1985     family => {
1986         type => 'string',
1987         pattern => '\S+',
1988         format_description => 'string',
1989         description => "Set SMBIOS1 family string.",
1990         optional => 1,
1991     },
1992 };
1993
1994 sub parse_smbios1 {
1995     my ($data) = @_;
1996
1997     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
1998     warn $@ if $@;
1999     return $res;
2000 }
2001
2002 sub print_smbios1 {
2003     my ($smbios1) = @_;
2004     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2005 }
2006
2007 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2008
2009 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2010 sub verify_bootdisk {
2011     my ($value, $noerr) = @_;
2012
2013     return $value if is_valid_drivename($value);
2014
2015     return undef if $noerr;
2016
2017     die "invalid boot disk '$value'\n";
2018 }
2019
2020 sub parse_watchdog {
2021     my ($value) = @_;
2022
2023     return undef if !$value;
2024
2025     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2026     warn $@ if $@;
2027     return $res;
2028 }
2029
2030 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2031 sub verify_usb_device {
2032     my ($value, $noerr) = @_;
2033
2034     return $value if parse_usb_device($value);
2035
2036     return undef if $noerr;
2037
2038     die "unable to parse usb device\n";
2039 }
2040
2041 # add JSON properties for create and set function
2042 sub json_config_properties {
2043     my $prop = shift;
2044
2045     foreach my $opt (keys %$confdesc) {
2046         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2047         $prop->{$opt} = $confdesc->{$opt};
2048     }
2049
2050     return $prop;
2051 }
2052
2053 sub check_type {
2054     my ($key, $value) = @_;
2055
2056     die "unknown setting '$key'\n" if !$confdesc->{$key};
2057
2058     my $type = $confdesc->{$key}->{type};
2059
2060     if (!defined($value)) {
2061         die "got undefined value\n";
2062     }
2063
2064     if ($value =~ m/[\n\r]/) {
2065         die "property contains a line feed\n";
2066     }
2067
2068     if ($type eq 'boolean') {
2069         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2070         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2071         die "type check ('boolean') failed - got '$value'\n";
2072     } elsif ($type eq 'integer') {
2073         return int($1) if $value =~ m/^(\d+)$/;
2074         die "type check ('integer') failed - got '$value'\n";
2075     } elsif ($type eq 'number') {
2076         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2077         die "type check ('number') failed - got '$value'\n";
2078     } elsif ($type eq 'string') {
2079         if (my $fmt = $confdesc->{$key}->{format}) {
2080             PVE::JSONSchema::check_format($fmt, $value);
2081             return $value;
2082         }
2083         $value =~ s/^\"(.*)\"$/$1/;
2084         return $value;
2085     } else {
2086         die "internal error"
2087     }
2088 }
2089
2090 sub check_iommu_support{
2091     #fixme : need to check IOMMU support
2092     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2093
2094     my $iommu=1;
2095     return $iommu;
2096
2097 }
2098
2099 sub touch_config {
2100     my ($vmid) = @_;
2101
2102     my $conf = PVE::QemuConfig->config_file($vmid);
2103     utime undef, undef, $conf;
2104 }
2105
2106 sub destroy_vm {
2107     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2108
2109     my $conffile = PVE::QemuConfig->config_file($vmid);
2110
2111     my $conf = PVE::QemuConfig->load_config($vmid);
2112
2113     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2114
2115     if ($conf->{template}) {
2116         # check if any base image is still used by a linked clone
2117         foreach_drive($conf, sub {
2118                 my ($ds, $drive) = @_;
2119
2120                 return if drive_is_cdrom($drive);
2121
2122                 my $volid = $drive->{file};
2123
2124                 return if !$volid || $volid =~ m|^/|;
2125
2126                 die "base volume '$volid' is still in use by linked cloned\n"
2127                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2128
2129         });
2130     }
2131
2132     # only remove disks owned by this VM
2133     foreach_drive($conf, sub {
2134         my ($ds, $drive) = @_;
2135
2136         return if drive_is_cdrom($drive);
2137
2138         my $volid = $drive->{file};
2139
2140         return if !$volid || $volid =~ m|^/|;
2141
2142         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2143         return if !$path || !$owner || ($owner != $vmid);
2144
2145         eval {
2146             PVE::Storage::vdisk_free($storecfg, $volid);
2147         };
2148         warn "Could not remove disk '$volid', check manually: $@" if $@;
2149
2150     });
2151
2152     if ($keep_empty_config) {
2153         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2154     } else {
2155         unlink $conffile;
2156     }
2157
2158     # also remove unused disk
2159     eval {
2160         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2161
2162         eval {
2163             PVE::Storage::foreach_volid($dl, sub {
2164                 my ($volid, $sid, $volname, $d) = @_;
2165                 PVE::Storage::vdisk_free($storecfg, $volid);
2166             });
2167         };
2168         warn $@ if $@;
2169
2170     };
2171     warn $@ if $@;
2172 }
2173
2174 sub parse_vm_config {
2175     my ($filename, $raw) = @_;
2176
2177     return undef if !defined($raw);
2178
2179     my $res = {
2180         digest => Digest::SHA::sha1_hex($raw),
2181         snapshots => {},
2182         pending => {},
2183     };
2184
2185     $filename =~ m|/qemu-server/(\d+)\.conf$|
2186         || die "got strange filename '$filename'";
2187
2188     my $vmid = $1;
2189
2190     my $conf = $res;
2191     my $descr;
2192     my $section = '';
2193
2194     my @lines = split(/\n/, $raw);
2195     foreach my $line (@lines) {
2196         next if $line =~ m/^\s*$/;
2197
2198         if ($line =~ m/^\[PENDING\]\s*$/i) {
2199             $section = 'pending';
2200             if (defined($descr)) {
2201                 $descr =~ s/\s+$//;
2202                 $conf->{description} = $descr;
2203             }
2204             $descr = undef;
2205             $conf = $res->{$section} = {};
2206             next;
2207
2208         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2209             $section = $1;
2210             if (defined($descr)) {
2211                 $descr =~ s/\s+$//;
2212                 $conf->{description} = $descr;
2213             }
2214             $descr = undef;
2215             $conf = $res->{snapshots}->{$section} = {};
2216             next;
2217         }
2218
2219         if ($line =~ m/^\#(.*)\s*$/) {
2220             $descr = '' if !defined($descr);
2221             $descr .= PVE::Tools::decode_text($1) . "\n";
2222             next;
2223         }
2224
2225         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2226             $descr = '' if !defined($descr);
2227             $descr .= PVE::Tools::decode_text($2);
2228         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2229             $conf->{snapstate} = $1;
2230         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2231             my $key = $1;
2232             my $value = $2;
2233             $conf->{$key} = $value;
2234         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2235             my $value = $1;
2236             if ($section eq 'pending') {
2237                 $conf->{delete} = $value; # we parse this later
2238             } else {
2239                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2240             }
2241         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2242             my $key = $1;
2243             my $value = $2;
2244             eval { $value = check_type($key, $value); };
2245             if ($@) {
2246                 warn "vm $vmid - unable to parse value of '$key' - $@";
2247             } else {
2248                 $key = 'ide2' if $key eq 'cdrom';
2249                 my $fmt = $confdesc->{$key}->{format};
2250                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2251                     my $v = parse_drive($key, $value);
2252                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2253                         $v->{file} = $volid;
2254                         $value = print_drive($vmid, $v);
2255                     } else {
2256                         warn "vm $vmid - unable to parse value of '$key'\n";
2257                         next;
2258                     }
2259                 }
2260
2261                 $conf->{$key} = $value;
2262             }
2263         }
2264     }
2265
2266     if (defined($descr)) {
2267         $descr =~ s/\s+$//;
2268         $conf->{description} = $descr;
2269     }
2270     delete $res->{snapstate}; # just to be sure
2271
2272     return $res;
2273 }
2274
2275 sub write_vm_config {
2276     my ($filename, $conf) = @_;
2277
2278     delete $conf->{snapstate}; # just to be sure
2279
2280     if ($conf->{cdrom}) {
2281         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2282         $conf->{ide2} = $conf->{cdrom};
2283         delete $conf->{cdrom};
2284     }
2285
2286     # we do not use 'smp' any longer
2287     if ($conf->{sockets}) {
2288         delete $conf->{smp};
2289     } elsif ($conf->{smp}) {
2290         $conf->{sockets} = $conf->{smp};
2291         delete $conf->{cores};
2292         delete $conf->{smp};
2293     }
2294
2295     my $used_volids = {};
2296
2297     my $cleanup_config = sub {
2298         my ($cref, $pending, $snapname) = @_;
2299
2300         foreach my $key (keys %$cref) {
2301             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2302                 $key eq 'snapstate' || $key eq 'pending';
2303             my $value = $cref->{$key};
2304             if ($key eq 'delete') {
2305                 die "propertry 'delete' is only allowed in [PENDING]\n"
2306                     if !$pending;
2307                 # fixme: check syntax?
2308                 next;
2309             }
2310             eval { $value = check_type($key, $value); };
2311             die "unable to parse value of '$key' - $@" if $@;
2312
2313             $cref->{$key} = $value;
2314
2315             if (!$snapname && is_valid_drivename($key)) {
2316                 my $drive = parse_drive($key, $value);
2317                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2318             }
2319         }
2320     };
2321
2322     &$cleanup_config($conf);
2323
2324     &$cleanup_config($conf->{pending}, 1);
2325
2326     foreach my $snapname (keys %{$conf->{snapshots}}) {
2327         die "internal error" if $snapname eq 'pending';
2328         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2329     }
2330
2331     # remove 'unusedX' settings if we re-add a volume
2332     foreach my $key (keys %$conf) {
2333         my $value = $conf->{$key};
2334         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2335             delete $conf->{$key};
2336         }
2337     }
2338
2339     my $generate_raw_config = sub {
2340         my ($conf, $pending) = @_;
2341
2342         my $raw = '';
2343
2344         # add description as comment to top of file
2345         if (defined(my $descr = $conf->{description})) {
2346             if ($descr) {
2347                 foreach my $cl (split(/\n/, $descr)) {
2348                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2349                 }
2350             } else {
2351                 $raw .= "#\n" if $pending;
2352             }
2353         }
2354
2355         foreach my $key (sort keys %$conf) {
2356             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2357             $raw .= "$key: $conf->{$key}\n";
2358         }
2359         return $raw;
2360     };
2361
2362     my $raw = &$generate_raw_config($conf);
2363
2364     if (scalar(keys %{$conf->{pending}})){
2365         $raw .= "\n[PENDING]\n";
2366         $raw .= &$generate_raw_config($conf->{pending}, 1);
2367     }
2368
2369     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2370         $raw .= "\n[$snapname]\n";
2371         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2372     }
2373
2374     return $raw;
2375 }
2376
2377 sub load_defaults {
2378
2379     my $res = {};
2380
2381     # we use static defaults from our JSON schema configuration
2382     foreach my $key (keys %$confdesc) {
2383         if (defined(my $default = $confdesc->{$key}->{default})) {
2384             $res->{$key} = $default;
2385         }
2386     }
2387
2388     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2389     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2390
2391     return $res;
2392 }
2393
2394 sub config_list {
2395     my $vmlist = PVE::Cluster::get_vmlist();
2396     my $res = {};
2397     return $res if !$vmlist || !$vmlist->{ids};
2398     my $ids = $vmlist->{ids};
2399
2400     foreach my $vmid (keys %$ids) {
2401         my $d = $ids->{$vmid};
2402         next if !$d->{node} || $d->{node} ne $nodename;
2403         next if !$d->{type} || $d->{type} ne 'qemu';
2404         $res->{$vmid}->{exists} = 1;
2405     }
2406     return $res;
2407 }
2408
2409 # test if VM uses local resources (to prevent migration)
2410 sub check_local_resources {
2411     my ($conf, $noerr) = @_;
2412
2413     my $loc_res = 0;
2414
2415     $loc_res = 1 if $conf->{hostusb}; # old syntax
2416     $loc_res = 1 if $conf->{hostpci}; # old syntax
2417
2418     foreach my $k (keys %$conf) {
2419         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2420         # sockets are safe: they will recreated be on the target side post-migrate
2421         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2422         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2423     }
2424
2425     die "VM uses local resources\n" if $loc_res && !$noerr;
2426
2427     return $loc_res;
2428 }
2429
2430 # check if used storages are available on all nodes (use by migrate)
2431 sub check_storage_availability {
2432     my ($storecfg, $conf, $node) = @_;
2433
2434     foreach_drive($conf, sub {
2435         my ($ds, $drive) = @_;
2436
2437         my $volid = $drive->{file};
2438         return if !$volid;
2439
2440         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2441         return if !$sid;
2442
2443         # check if storage is available on both nodes
2444         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2445         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2446    });
2447 }
2448
2449 # list nodes where all VM images are available (used by has_feature API)
2450 sub shared_nodes {
2451     my ($conf, $storecfg) = @_;
2452
2453     my $nodelist = PVE::Cluster::get_nodelist();
2454     my $nodehash = { map { $_ => 1 } @$nodelist };
2455     my $nodename = PVE::INotify::nodename();
2456
2457     foreach_drive($conf, sub {
2458         my ($ds, $drive) = @_;
2459
2460         my $volid = $drive->{file};
2461         return if !$volid;
2462
2463         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2464         if ($storeid) {
2465             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2466             if ($scfg->{disable}) {
2467                 $nodehash = {};
2468             } elsif (my $avail = $scfg->{nodes}) {
2469                 foreach my $node (keys %$nodehash) {
2470                     delete $nodehash->{$node} if !$avail->{$node};
2471                 }
2472             } elsif (!$scfg->{shared}) {
2473                 foreach my $node (keys %$nodehash) {
2474                     delete $nodehash->{$node} if $node ne $nodename
2475                 }
2476             }
2477         }
2478     });
2479
2480     return $nodehash
2481 }
2482
2483 sub check_cmdline {
2484     my ($pidfile, $pid) = @_;
2485
2486     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2487     if (defined($fh)) {
2488         my $line = <$fh>;
2489         $fh->close;
2490         return undef if !$line;
2491         my @param = split(/\0/, $line);
2492
2493         my $cmd = $param[0];
2494         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2495
2496         for (my $i = 0; $i < scalar (@param); $i++) {
2497             my $p = $param[$i];
2498             next if !$p;
2499             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2500                 my $p = $param[$i+1];
2501                 return 1 if $p && ($p eq $pidfile);
2502                 return undef;
2503             }
2504         }
2505     }
2506     return undef;
2507 }
2508
2509 sub check_running {
2510     my ($vmid, $nocheck, $node) = @_;
2511
2512     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2513
2514     die "unable to find configuration file for VM $vmid - no such machine\n"
2515         if !$nocheck && ! -f $filename;
2516
2517     my $pidfile = pidfile_name($vmid);
2518
2519     if (my $fd = IO::File->new("<$pidfile")) {
2520         my $st = stat($fd);
2521         my $line = <$fd>;
2522         close($fd);
2523
2524         my $mtime = $st->mtime;
2525         if ($mtime > time()) {
2526             warn "file '$filename' modified in future\n";
2527         }
2528
2529         if ($line =~ m/^(\d+)$/) {
2530             my $pid = $1;
2531             if (check_cmdline($pidfile, $pid)) {
2532                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2533                     return $pid;
2534                 }
2535             }
2536         }
2537     }
2538
2539     return undef;
2540 }
2541
2542 sub vzlist {
2543
2544     my $vzlist = config_list();
2545
2546     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2547
2548     while (defined(my $de = $fd->read)) {
2549         next if $de !~ m/^(\d+)\.pid$/;
2550         my $vmid = $1;
2551         next if !defined($vzlist->{$vmid});
2552         if (my $pid = check_running($vmid)) {
2553             $vzlist->{$vmid}->{pid} = $pid;
2554         }
2555     }
2556
2557     return $vzlist;
2558 }
2559
2560 sub disksize {
2561     my ($storecfg, $conf) = @_;
2562
2563     my $bootdisk = $conf->{bootdisk};
2564     return undef if !$bootdisk;
2565     return undef if !is_valid_drivename($bootdisk);
2566
2567     return undef if !$conf->{$bootdisk};
2568
2569     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2570     return undef if !defined($drive);
2571
2572     return undef if drive_is_cdrom($drive);
2573
2574     my $volid = $drive->{file};
2575     return undef if !$volid;
2576
2577     return $drive->{size};
2578 }
2579
2580 my $last_proc_pid_stat;
2581
2582 # get VM status information
2583 # This must be fast and should not block ($full == false)
2584 # We only query KVM using QMP if $full == true (this can be slow)
2585 sub vmstatus {
2586     my ($opt_vmid, $full) = @_;
2587
2588     my $res = {};
2589
2590     my $storecfg = PVE::Storage::config();
2591
2592     my $list = vzlist();
2593     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2594
2595     my $cpucount = $cpuinfo->{cpus} || 1;
2596
2597     foreach my $vmid (keys %$list) {
2598         next if $opt_vmid && ($vmid ne $opt_vmid);
2599
2600         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2601         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2602
2603         my $d = {};
2604         $d->{pid} = $list->{$vmid}->{pid};
2605
2606         # fixme: better status?
2607         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2608
2609         my $size = disksize($storecfg, $conf);
2610         if (defined($size)) {
2611             $d->{disk} = 0; # no info available
2612             $d->{maxdisk} = $size;
2613         } else {
2614             $d->{disk} = 0;
2615             $d->{maxdisk} = 0;
2616         }
2617
2618         $d->{cpus} = ($conf->{sockets} || 1) * ($conf->{cores} || 1);
2619         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2620         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2621
2622         $d->{name} = $conf->{name} || "VM $vmid";
2623         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024) : 0;
2624
2625         if ($conf->{balloon}) {
2626             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2627             $d->{shares} = defined($conf->{shares}) ? $conf->{shares} : 1000;
2628         }
2629
2630         $d->{uptime} = 0;
2631         $d->{cpu} = 0;
2632         $d->{mem} = 0;
2633
2634         $d->{netout} = 0;
2635         $d->{netin} = 0;
2636
2637         $d->{diskread} = 0;
2638         $d->{diskwrite} = 0;
2639
2640         $d->{template} = PVE::QemuConfig->is_template($conf);
2641
2642         $res->{$vmid} = $d;
2643     }
2644
2645     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2646     foreach my $dev (keys %$netdev) {
2647         next if $dev !~ m/^tap([1-9]\d*)i/;
2648         my $vmid = $1;
2649         my $d = $res->{$vmid};
2650         next if !$d;
2651
2652         $d->{netout} += $netdev->{$dev}->{receive};
2653         $d->{netin} += $netdev->{$dev}->{transmit};
2654
2655         if ($full) {
2656             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2657             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2658         }
2659
2660     }
2661
2662     my $ctime = gettimeofday;
2663
2664     foreach my $vmid (keys %$list) {
2665
2666         my $d = $res->{$vmid};
2667         my $pid = $d->{pid};
2668         next if !$pid;
2669
2670         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2671         next if !$pstat; # not running
2672
2673         my $used = $pstat->{utime} + $pstat->{stime};
2674
2675         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2676
2677         if ($pstat->{vsize}) {
2678             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2679         }
2680
2681         my $old = $last_proc_pid_stat->{$pid};
2682         if (!$old) {
2683             $last_proc_pid_stat->{$pid} = {
2684                 time => $ctime,
2685                 used => $used,
2686                 cpu => 0,
2687             };
2688             next;
2689         }
2690
2691         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2692
2693         if ($dtime > 1000) {
2694             my $dutime = $used -  $old->{used};
2695
2696             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2697             $last_proc_pid_stat->{$pid} = {
2698                 time => $ctime,
2699                 used => $used,
2700                 cpu => $d->{cpu},
2701             };
2702         } else {
2703             $d->{cpu} = $old->{cpu};
2704         }
2705     }
2706
2707     return $res if !$full;
2708
2709     my $qmpclient = PVE::QMPClient->new();
2710
2711     my $ballooncb = sub {
2712         my ($vmid, $resp) = @_;
2713
2714         my $info = $resp->{'return'};
2715         return if !$info->{max_mem};
2716
2717         my $d = $res->{$vmid};
2718
2719         # use memory assigned to VM
2720         $d->{maxmem} = $info->{max_mem};
2721         $d->{balloon} = $info->{actual};
2722
2723         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2724             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2725             $d->{freemem} = $info->{free_mem};
2726         }
2727
2728         $d->{ballooninfo} = $info;
2729     };
2730
2731     my $blockstatscb = sub {
2732         my ($vmid, $resp) = @_;
2733         my $data = $resp->{'return'} || [];
2734         my $totalrdbytes = 0;
2735         my $totalwrbytes = 0;
2736
2737         for my $blockstat (@$data) {
2738             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2739             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2740
2741             $blockstat->{device} =~ s/drive-//;
2742             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2743         }
2744         $res->{$vmid}->{diskread} = $totalrdbytes;
2745         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2746     };
2747
2748     my $statuscb = sub {
2749         my ($vmid, $resp) = @_;
2750
2751         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2752         # this fails if ballon driver is not loaded, so this must be
2753         # the last commnand (following command are aborted if this fails).
2754         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2755
2756         my $status = 'unknown';
2757         if (!defined($status = $resp->{'return'}->{status})) {
2758             warn "unable to get VM status\n";
2759             return;
2760         }
2761
2762         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2763     };
2764
2765     foreach my $vmid (keys %$list) {
2766         next if $opt_vmid && ($vmid ne $opt_vmid);
2767         next if !$res->{$vmid}->{pid}; # not running
2768         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2769     }
2770
2771     $qmpclient->queue_execute(undef, 2);
2772
2773     foreach my $vmid (keys %$list) {
2774         next if $opt_vmid && ($vmid ne $opt_vmid);
2775         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2776     }
2777
2778     return $res;
2779 }
2780
2781 sub foreach_drive {
2782     my ($conf, $func, @param) = @_;
2783
2784     foreach my $ds (valid_drive_names()) {
2785         next if !defined($conf->{$ds});
2786
2787         my $drive = parse_drive($ds, $conf->{$ds});
2788         next if !$drive;
2789
2790         &$func($ds, $drive, @param);
2791     }
2792 }
2793
2794 sub foreach_volid {
2795     my ($conf, $func, @param) = @_;
2796
2797     my $volhash = {};
2798
2799     my $test_volid = sub {
2800         my ($volid, $is_cdrom, $replicate, $snapname) = @_;
2801
2802         return if !$volid;
2803
2804         $volhash->{$volid}->{cdrom} //= 1;
2805         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2806
2807         $volhash->{$volid}->{replicate} //= 0;
2808         $volhash->{$volid}->{replicate} = 1 if $replicate;
2809
2810         $volhash->{$volid}->{referenced_in_config} //= 0;
2811         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
2812
2813         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
2814             if defined($snapname);
2815     };
2816
2817     foreach_drive($conf, sub {
2818         my ($ds, $drive) = @_;
2819         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, undef);
2820     });
2821
2822     foreach my $snapname (keys %{$conf->{snapshots}}) {
2823         my $snap = $conf->{snapshots}->{$snapname};
2824         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
2825         foreach_drive($snap, sub {
2826             my ($ds, $drive) = @_;
2827             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $snapname);
2828         });
2829     }
2830
2831     foreach my $volid (keys %$volhash) {
2832         &$func($volid, $volhash->{$volid}, @param);
2833     }
2834 }
2835
2836 sub vga_conf_has_spice {
2837     my ($vga) = @_;
2838
2839     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2840
2841     return $1 || 1;
2842 }
2843
2844 sub config_to_command {
2845     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2846
2847     my $cmd = [];
2848     my $globalFlags = [];
2849     my $machineFlags = [];
2850     my $rtcFlags = [];
2851     my $cpuFlags = [];
2852     my $devices = [];
2853     my $pciaddr = '';
2854     my $bridges = {};
2855     my $kvmver = kvm_user_version();
2856     my $vernum = 0; # unknown
2857     my $ostype = $conf->{ostype};
2858     my $winversion = windows_version($ostype);
2859     my $kvm = $conf->{kvm} // 1;
2860
2861     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
2862
2863     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2864         $vernum = $1*1000000+$2*1000;
2865     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2866         $vernum = $1*1000000+$2*1000+$3;
2867     }
2868
2869     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2870
2871     my $have_ovz = -f '/proc/vz/vestat';
2872
2873     my $q35 = machine_type_is_q35($conf);
2874     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2875     my $machine_type = $forcemachine || $conf->{machine};
2876     my $use_old_bios_files = undef;
2877     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2878
2879     my $cpuunits = defined($conf->{cpuunits}) ?
2880             $conf->{cpuunits} : $defaults->{cpuunits};
2881
2882     push @$cmd, '/usr/bin/kvm';
2883
2884     push @$cmd, '-id', $vmid;
2885
2886     my $use_virtio = 0;
2887
2888     my $qmpsocket = qmp_socket($vmid);
2889     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2890     push @$cmd, '-mon', "chardev=qmp,mode=control";
2891
2892
2893     push @$cmd, '-pidfile' , pidfile_name($vmid);
2894
2895     push @$cmd, '-daemonize';
2896
2897     if ($conf->{smbios1}) {
2898         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2899     }
2900
2901     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2902         die "uefi base image not found\n" if ! -f $OVMF_CODE;
2903
2904         my $path;
2905         my $format = 'raw';
2906         if (my $efidisk = $conf->{efidisk0}) {
2907             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
2908             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2909             if ($storeid) {
2910                 $path = PVE::Storage::path($storecfg, $d->{file});
2911             } else {
2912                 $path = $d->{file};
2913             }
2914             $format = $d->{format} if $d->{format};
2915         } else {
2916             warn "no efidisk configured! Using temporary efivars disk.\n";
2917             $path = "/tmp/$vmid-ovmf.fd";
2918             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
2919         }
2920
2921         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
2922         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
2923     }
2924
2925
2926     # add usb controllers
2927     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2928     push @$devices, @usbcontrollers if @usbcontrollers;
2929     my $vga = $conf->{vga};
2930
2931     my $qxlnum = vga_conf_has_spice($vga);
2932     $vga = 'qxl' if $qxlnum;
2933
2934     if (!$vga) {
2935         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
2936             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
2937         } else {
2938             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
2939         }
2940     }
2941
2942     # enable absolute mouse coordinates (needed by vnc)
2943     my $tablet;
2944     if (defined($conf->{tablet})) {
2945         $tablet = $conf->{tablet};
2946     } else {
2947         $tablet = $defaults->{tablet};
2948         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2949         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2950     }
2951
2952     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2953
2954     my $kvm_off = 0;
2955     my $gpu_passthrough;
2956
2957     # host pci devices
2958     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2959         my $d = parse_hostpci($conf->{"hostpci$i"});
2960         next if !$d;
2961
2962         my $pcie = $d->{pcie};
2963         if($pcie){
2964             die "q35 machine model is not enabled" if !$q35;
2965             $pciaddr = print_pcie_addr("hostpci$i");
2966         }else{
2967             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2968         }
2969
2970         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2971         my $romfile = $d->{romfile};
2972
2973         my $xvga = '';
2974         if ($d->{'x-vga'}) {
2975             $xvga = ',x-vga=on';
2976             $kvm_off = 1;
2977             $vga = 'none';
2978             $gpu_passthrough = 1;
2979
2980             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2981                 $xvga = "";
2982             }
2983         }
2984         my $pcidevices = $d->{pciid};
2985         my $multifunction = 1 if @$pcidevices > 1;
2986
2987         my $j=0;
2988         foreach my $pcidevice (@$pcidevices) {
2989
2990             my $id = "hostpci$i";
2991             $id .= ".$j" if $multifunction;
2992             my $addr = $pciaddr;
2993             $addr .= ".$j" if $multifunction;
2994             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
2995
2996             if($j == 0){
2997                 $devicestr .= "$rombar$xvga";
2998                 $devicestr .= ",multifunction=on" if $multifunction;
2999                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3000             }
3001
3002             push @$devices, '-device', $devicestr;
3003             $j++;
3004         }
3005     }
3006
3007     # usb devices
3008     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3009     push @$devices, @usbdevices if @usbdevices;
3010     # serial devices
3011     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3012         if (my $path = $conf->{"serial$i"}) {
3013             if ($path eq 'socket') {
3014                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3015                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3016                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3017             } else {
3018                 die "no such serial device\n" if ! -c $path;
3019                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3020                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3021             }
3022         }
3023     }
3024
3025     # parallel devices
3026     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3027         if (my $path = $conf->{"parallel$i"}) {
3028             die "no such parallel device\n" if ! -c $path;
3029             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3030             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3031             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3032         }
3033     }
3034
3035     my $vmname = $conf->{name} || "vm$vmid";
3036
3037     push @$cmd, '-name', $vmname;
3038
3039     my $sockets = 1;
3040     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3041     $sockets = $conf->{sockets} if  $conf->{sockets};
3042
3043     my $cores = $conf->{cores} || 1;
3044
3045     my $maxcpus = $sockets * $cores;
3046
3047     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3048
3049     my $allowed_vcpus = $cpuinfo->{cpus};
3050
3051     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3052         if ($allowed_vcpus < $maxcpus);
3053
3054     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3055
3056         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3057         for (my $i = 2; $i <= $vcpus; $i++)  {
3058             my $cpustr = print_cpu_device($conf,$i);
3059             push @$cmd, '-device', $cpustr;
3060         }
3061
3062     } else {
3063
3064         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3065     }
3066     push @$cmd, '-nodefaults';
3067
3068     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3069
3070     my $bootindex_hash = {};
3071     my $i = 1;
3072     foreach my $o (split(//, $bootorder)) {
3073         $bootindex_hash->{$o} = $i*100;
3074         $i++;
3075     }
3076
3077     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3078
3079     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3080
3081     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3082
3083     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3084
3085     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3086         my $socket = vnc_socket($vmid);
3087         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3088     } else {
3089         push @$cmd, '-nographic';
3090     }
3091
3092     # time drift fix
3093     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3094
3095     my $useLocaltime = $conf->{localtime};
3096
3097     if ($winversion >= 5) { # windows
3098         $useLocaltime = 1 if !defined($conf->{localtime});
3099
3100         # use time drift fix when acpi is enabled
3101         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3102             $tdf = 1 if !defined($conf->{tdf});
3103         }
3104     }
3105
3106     if ($winversion >= 6) {
3107         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3108         push @$cmd, '-no-hpet';
3109     }
3110
3111     push @$rtcFlags, 'driftfix=slew' if $tdf;
3112
3113     if (!$kvm) {
3114         push @$machineFlags, 'accel=tcg';
3115     }
3116
3117     if ($machine_type) {
3118         push @$machineFlags, "type=${machine_type}";
3119     }
3120
3121     if ($conf->{startdate}) {
3122         push @$rtcFlags, "base=$conf->{startdate}";
3123     } elsif ($useLocaltime) {
3124         push @$rtcFlags, 'base=localtime';
3125     }
3126
3127     my $cpu = $kvm ? "kvm64" : "qemu64";
3128     if (my $cputype = $conf->{cpu}) {
3129         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3130             or die "Cannot parse cpu description: $cputype\n";
3131         $cpu = $cpuconf->{cputype};
3132         $kvm_off = 1 if $cpuconf->{hidden};
3133     }
3134
3135     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3136
3137     push @$cpuFlags , '-x2apic'
3138         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3139
3140     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3141
3142     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3143
3144     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3145
3146         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3147         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3148     }
3149
3150     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3151
3152     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3153
3154     push @$cpuFlags, 'kvm=off' if $kvm_off;
3155
3156     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3157         die "internal error"; # should not happen
3158
3159     push @$cpuFlags, "vendor=${cpu_vendor}"
3160         if $cpu_vendor ne 'default';
3161
3162     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3163
3164     push @$cmd, '-cpu', $cpu;
3165
3166     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3167
3168     push @$cmd, '-S' if $conf->{freeze};
3169
3170     # set keyboard layout
3171     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3172     push @$cmd, '-k', $kb if $kb;
3173
3174     # enable sound
3175     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3176     #push @$cmd, '-soundhw', 'es1370';
3177     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3178
3179     if($conf->{agent}) {
3180         my $qgasocket = qmp_socket($vmid, 1);
3181         my $pciaddr = print_pci_addr("qga0", $bridges);
3182         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3183         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3184         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3185     }
3186
3187     my $spice_port;
3188
3189     if ($qxlnum) {
3190         if ($qxlnum > 1) {
3191             if ($winversion){
3192                 for(my $i = 1; $i < $qxlnum; $i++){
3193                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3194                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3195                 }
3196             } else {
3197                 # assume other OS works like Linux
3198                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3199                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3200             }
3201         }
3202
3203         my $pciaddr = print_pci_addr("spice", $bridges);
3204
3205         my $nodename = PVE::INotify::nodename();
3206         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3207         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3208         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3209         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3210         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3211
3212         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3213
3214         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3215         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3216         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3217     }
3218
3219     # enable balloon by default, unless explicitly disabled
3220     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3221         $pciaddr = print_pci_addr("balloon0", $bridges);
3222         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3223     }
3224
3225     if ($conf->{watchdog}) {
3226         my $wdopts = parse_watchdog($conf->{watchdog});
3227         $pciaddr = print_pci_addr("watchdog", $bridges);
3228         my $watchdog = $wdopts->{model} || 'i6300esb';
3229         push @$devices, '-device', "$watchdog$pciaddr";
3230         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3231     }
3232
3233     my $vollist = [];
3234     my $scsicontroller = {};
3235     my $ahcicontroller = {};
3236     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3237
3238     # Add iscsi initiator name if available
3239     if (my $initiator = get_initiator_name()) {
3240         push @$devices, '-iscsi', "initiator-name=$initiator";
3241     }
3242
3243     foreach_drive($conf, sub {
3244         my ($ds, $drive) = @_;
3245
3246         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3247             push @$vollist, $drive->{file};
3248         }
3249
3250         # ignore efidisk here, already added in bios/fw handling code above
3251         return if $drive->{interface} eq 'efidisk';
3252
3253         $use_virtio = 1 if $ds =~ m/^virtio/;
3254
3255         if (drive_is_cdrom ($drive)) {
3256             if ($bootindex_hash->{d}) {
3257                 $drive->{bootindex} = $bootindex_hash->{d};
3258                 $bootindex_hash->{d} += 1;
3259             }
3260         } else {
3261             if ($bootindex_hash->{c}) {
3262                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3263                 $bootindex_hash->{c} += 1;
3264             }
3265         }
3266
3267         if($drive->{interface} eq 'virtio'){
3268            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3269         }
3270
3271         if ($drive->{interface} eq 'scsi') {
3272
3273             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3274
3275             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3276             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3277
3278             my $iothread = '';
3279             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3280                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3281                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3282             } elsif ($drive->{iothread}) {
3283                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3284             }
3285
3286             my $queues = '';
3287             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3288                 $queues = ",num_queues=$drive->{queues}";
3289             }
3290
3291             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3292             $scsicontroller->{$controller}=1;
3293         }
3294
3295         if ($drive->{interface} eq 'sata') {
3296            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3297            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3298            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3299            $ahcicontroller->{$controller}=1;
3300         }
3301
3302         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3303         push @$devices, '-drive',$drive_cmd;
3304         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3305     });
3306
3307     for (my $i = 0; $i < $MAX_NETS; $i++) {
3308          next if !$conf->{"net$i"};
3309          my $d = parse_net($conf->{"net$i"});
3310          next if !$d;
3311
3312          $use_virtio = 1 if $d->{model} eq 'virtio';
3313
3314          if ($bootindex_hash->{n}) {
3315             $d->{bootindex} = $bootindex_hash->{n};
3316             $bootindex_hash->{n} += 1;
3317          }
3318
3319          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3320          push @$devices, '-netdev', $netdevfull;
3321
3322          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3323          push @$devices, '-device', $netdevicefull;
3324     }
3325
3326     if (!$q35) {
3327         # add pci bridges
3328         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3329            $bridges->{1} = 1;
3330            $bridges->{2} = 1;
3331         }
3332
3333         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3334
3335         while (my ($k, $v) = each %$bridges) {
3336             $pciaddr = print_pci_addr("pci.$k");
3337             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3338         }
3339     }
3340
3341     # add custom args
3342     if ($conf->{args}) {
3343         my $aa = PVE::Tools::split_args($conf->{args});
3344         push @$cmd, @$aa;
3345     }
3346
3347     push @$cmd, @$devices;
3348     push @$cmd, '-rtc', join(',', @$rtcFlags)
3349         if scalar(@$rtcFlags);
3350     push @$cmd, '-machine', join(',', @$machineFlags)
3351         if scalar(@$machineFlags);
3352     push @$cmd, '-global', join(',', @$globalFlags)
3353         if scalar(@$globalFlags);
3354
3355     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3356 }
3357
3358 sub vnc_socket {
3359     my ($vmid) = @_;
3360     return "${var_run_tmpdir}/$vmid.vnc";
3361 }
3362
3363 sub spice_port {
3364     my ($vmid) = @_;
3365
3366     my $res = vm_mon_cmd($vmid, 'query-spice');
3367
3368     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3369 }
3370
3371 sub qmp_socket {
3372     my ($vmid, $qga) = @_;
3373     my $sockettype = $qga ? 'qga' : 'qmp';
3374     return "${var_run_tmpdir}/$vmid.$sockettype";
3375 }
3376
3377 sub pidfile_name {
3378     my ($vmid) = @_;
3379     return "${var_run_tmpdir}/$vmid.pid";
3380 }
3381
3382 sub vm_devices_list {
3383     my ($vmid) = @_;
3384
3385     my $res = vm_mon_cmd($vmid, 'query-pci');
3386     my $devices = {};
3387     foreach my $pcibus (@$res) {
3388         foreach my $device (@{$pcibus->{devices}}) {
3389             next if !$device->{'qdev_id'};
3390             if ($device->{'pci_bridge'}) {
3391                 $devices->{$device->{'qdev_id'}} = 1;
3392                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3393                     next if !$bridge_device->{'qdev_id'};
3394                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3395                     $devices->{$device->{'qdev_id'}}++;
3396                 }
3397             } else {
3398                 $devices->{$device->{'qdev_id'}} = 1;
3399             }
3400         }
3401     }
3402
3403     my $resblock = vm_mon_cmd($vmid, 'query-block');
3404     foreach my $block (@$resblock) {
3405         if($block->{device} =~ m/^drive-(\S+)/){
3406                 $devices->{$1} = 1;
3407         }
3408     }
3409
3410     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3411     foreach my $mice (@$resmice) {
3412         if ($mice->{name} eq 'QEMU HID Tablet') {
3413             $devices->{tablet} = 1;
3414             last;
3415         }
3416     }
3417
3418     # for usb devices there is no query-usb
3419     # but we can iterate over the entries in
3420     # qom-list path=/machine/peripheral
3421     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3422     foreach my $per (@$resperipheral) {
3423         if ($per->{name} =~ m/^usb\d+$/) {
3424             $devices->{$per->{name}} = 1;
3425         }
3426     }
3427
3428     return $devices;
3429 }
3430
3431 sub vm_deviceplug {
3432     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3433
3434     my $q35 = machine_type_is_q35($conf);
3435
3436     my $devices_list = vm_devices_list($vmid);
3437     return 1 if defined($devices_list->{$deviceid});
3438
3439     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3440
3441     if ($deviceid eq 'tablet') {
3442
3443         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3444
3445     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3446
3447         die "usb hotplug currently not reliable\n";
3448         # since we can't reliably hot unplug all added usb devices
3449         # and usb passthrough disables live migration
3450         # we disable usb hotplugging for now
3451         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3452
3453     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3454
3455         qemu_iothread_add($vmid, $deviceid, $device);
3456
3457         qemu_driveadd($storecfg, $vmid, $device);
3458         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3459
3460         qemu_deviceadd($vmid, $devicefull);
3461         eval { qemu_deviceaddverify($vmid, $deviceid); };
3462         if (my $err = $@) {
3463             eval { qemu_drivedel($vmid, $deviceid); };
3464             warn $@ if $@;
3465             die $err;
3466         }
3467
3468     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3469
3470
3471         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3472         my $pciaddr = print_pci_addr($deviceid);
3473         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3474
3475         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3476
3477         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3478             qemu_iothread_add($vmid, $deviceid, $device);
3479             $devicefull .= ",iothread=iothread-$deviceid";
3480         }
3481
3482         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3483             $devicefull .= ",num_queues=$device->{queues}";
3484         }
3485
3486         qemu_deviceadd($vmid, $devicefull);
3487         qemu_deviceaddverify($vmid, $deviceid);
3488
3489     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3490
3491         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3492         qemu_driveadd($storecfg, $vmid, $device);
3493
3494         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3495         eval { qemu_deviceadd($vmid, $devicefull); };
3496         if (my $err = $@) {
3497             eval { qemu_drivedel($vmid, $deviceid); };
3498             warn $@ if $@;
3499             die $err;
3500         }
3501
3502     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3503
3504         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3505
3506         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3507         my $use_old_bios_files = undef;
3508         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3509
3510         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3511         qemu_deviceadd($vmid, $netdevicefull);
3512         eval { qemu_deviceaddverify($vmid, $deviceid); };
3513         if (my $err = $@) {
3514             eval { qemu_netdevdel($vmid, $deviceid); };
3515             warn $@ if $@;
3516             die $err;
3517         }
3518
3519     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3520
3521         my $bridgeid = $2;
3522         my $pciaddr = print_pci_addr($deviceid);
3523         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3524
3525         qemu_deviceadd($vmid, $devicefull);
3526         qemu_deviceaddverify($vmid, $deviceid);
3527
3528     } else {
3529         die "can't hotplug device '$deviceid'\n";
3530     }
3531
3532     return 1;
3533 }
3534
3535 # fixme: this should raise exceptions on error!
3536 sub vm_deviceunplug {
3537     my ($vmid, $conf, $deviceid) = @_;
3538
3539     my $devices_list = vm_devices_list($vmid);
3540     return 1 if !defined($devices_list->{$deviceid});
3541
3542     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3543
3544     if ($deviceid eq 'tablet') {
3545
3546         qemu_devicedel($vmid, $deviceid);
3547
3548     } elsif ($deviceid =~ m/^usb\d+$/) {
3549
3550         die "usb hotplug currently not reliable\n";
3551         # when unplugging usb devices this way,
3552         # there may be remaining usb controllers/hubs
3553         # so we disable it for now
3554         qemu_devicedel($vmid, $deviceid);
3555         qemu_devicedelverify($vmid, $deviceid);
3556
3557     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3558
3559         qemu_devicedel($vmid, $deviceid);
3560         qemu_devicedelverify($vmid, $deviceid);
3561         qemu_drivedel($vmid, $deviceid);
3562         qemu_iothread_del($conf, $vmid, $deviceid);
3563
3564     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3565
3566         qemu_devicedel($vmid, $deviceid);
3567         qemu_devicedelverify($vmid, $deviceid);
3568         qemu_iothread_del($conf, $vmid, $deviceid);
3569
3570     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3571
3572         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3573         my $device = parse_drive($deviceid, $conf->{$deviceid});
3574         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3575
3576         qemu_devicedel($vmid, $deviceid);
3577         qemu_drivedel($vmid, $deviceid);
3578         qemu_deletescsihw($conf, $vmid, $deviceid);
3579
3580     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3581
3582         qemu_devicedel($vmid, $deviceid);
3583         qemu_devicedelverify($vmid, $deviceid);
3584         qemu_netdevdel($vmid, $deviceid);
3585
3586     } else {
3587         die "can't unplug device '$deviceid'\n";
3588     }
3589
3590     return 1;
3591 }
3592
3593 sub qemu_deviceadd {
3594     my ($vmid, $devicefull) = @_;
3595
3596     $devicefull = "driver=".$devicefull;
3597     my %options =  split(/[=,]/, $devicefull);
3598
3599     vm_mon_cmd($vmid, "device_add" , %options);
3600 }
3601
3602 sub qemu_devicedel {
3603     my ($vmid, $deviceid) = @_;
3604
3605     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3606 }
3607
3608 sub qemu_iothread_add {
3609     my($vmid, $deviceid, $device) = @_;
3610
3611     if ($device->{iothread}) {
3612         my $iothreads = vm_iothreads_list($vmid);
3613         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3614     }
3615 }
3616
3617 sub qemu_iothread_del {
3618     my($conf, $vmid, $deviceid) = @_;
3619
3620     my $device = parse_drive($deviceid, $conf->{$deviceid});
3621     if ($device->{iothread}) {
3622         my $iothreads = vm_iothreads_list($vmid);
3623         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3624     }
3625 }
3626
3627 sub qemu_objectadd {
3628     my($vmid, $objectid, $qomtype) = @_;
3629
3630     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3631
3632     return 1;
3633 }
3634
3635 sub qemu_objectdel {
3636     my($vmid, $objectid) = @_;
3637
3638     vm_mon_cmd($vmid, "object-del", id => $objectid);
3639
3640     return 1;
3641 }
3642
3643 sub qemu_driveadd {
3644     my ($storecfg, $vmid, $device) = @_;
3645
3646     my $drive = print_drive_full($storecfg, $vmid, $device);
3647     $drive =~ s/\\/\\\\/g;
3648     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3649
3650     # If the command succeeds qemu prints: "OK"
3651     return 1 if $ret =~ m/OK/s;
3652
3653     die "adding drive failed: $ret\n";
3654 }
3655
3656 sub qemu_drivedel {
3657     my($vmid, $deviceid) = @_;
3658
3659     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3660     $ret =~ s/^\s+//;
3661
3662     return 1 if $ret eq "";
3663
3664     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3665     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3666
3667     die "deleting drive $deviceid failed : $ret\n";
3668 }
3669
3670 sub qemu_deviceaddverify {
3671     my ($vmid, $deviceid) = @_;
3672
3673     for (my $i = 0; $i <= 5; $i++) {
3674          my $devices_list = vm_devices_list($vmid);
3675          return 1 if defined($devices_list->{$deviceid});
3676          sleep 1;
3677     }
3678
3679     die "error on hotplug device '$deviceid'\n";
3680 }
3681
3682
3683 sub qemu_devicedelverify {
3684     my ($vmid, $deviceid) = @_;
3685
3686     # need to verify that the device is correctly removed as device_del
3687     # is async and empty return is not reliable
3688
3689     for (my $i = 0; $i <= 5; $i++) {
3690          my $devices_list = vm_devices_list($vmid);
3691          return 1 if !defined($devices_list->{$deviceid});
3692          sleep 1;
3693     }
3694
3695     die "error on hot-unplugging device '$deviceid'\n";
3696 }
3697
3698 sub qemu_findorcreatescsihw {
3699     my ($storecfg, $conf, $vmid, $device) = @_;
3700
3701     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3702
3703     my $scsihwid="$controller_prefix$controller";
3704     my $devices_list = vm_devices_list($vmid);
3705
3706     if(!defined($devices_list->{$scsihwid})) {
3707         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3708     }
3709
3710     return 1;
3711 }
3712
3713 sub qemu_deletescsihw {
3714     my ($conf, $vmid, $opt) = @_;
3715
3716     my $device = parse_drive($opt, $conf->{$opt});
3717
3718     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3719         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3720         return 1;
3721     }
3722
3723     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3724
3725     my $devices_list = vm_devices_list($vmid);
3726     foreach my $opt (keys %{$devices_list}) {
3727         if (PVE::QemuServer::is_valid_drivename($opt)) {
3728             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3729             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3730                 return 1;
3731             }
3732         }
3733     }
3734
3735     my $scsihwid="scsihw$controller";
3736
3737     vm_deviceunplug($vmid, $conf, $scsihwid);
3738
3739     return 1;
3740 }
3741
3742 sub qemu_add_pci_bridge {
3743     my ($storecfg, $conf, $vmid, $device) = @_;
3744
3745     my $bridges = {};
3746
3747     my $bridgeid;
3748
3749     print_pci_addr($device, $bridges);
3750
3751     while (my ($k, $v) = each %$bridges) {
3752         $bridgeid = $k;
3753     }
3754     return 1 if !defined($bridgeid) || $bridgeid < 1;
3755
3756     my $bridge = "pci.$bridgeid";
3757     my $devices_list = vm_devices_list($vmid);
3758
3759     if (!defined($devices_list->{$bridge})) {
3760         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3761     }
3762
3763     return 1;
3764 }
3765
3766 sub qemu_set_link_status {
3767     my ($vmid, $device, $up) = @_;
3768
3769     vm_mon_cmd($vmid, "set_link", name => $device,
3770                up => $up ? JSON::true : JSON::false);
3771 }
3772
3773 sub qemu_netdevadd {
3774     my ($vmid, $conf, $device, $deviceid) = @_;
3775
3776     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3777     my %options =  split(/[=,]/, $netdev);
3778
3779     vm_mon_cmd($vmid, "netdev_add",  %options);
3780     return 1;
3781 }
3782
3783 sub qemu_netdevdel {
3784     my ($vmid, $deviceid) = @_;
3785
3786     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3787 }
3788
3789 sub qemu_usb_hotplug {
3790     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3791
3792     return if !$device;
3793
3794     # remove the old one first
3795     vm_deviceunplug($vmid, $conf, $deviceid);
3796
3797     # check if xhci controller is necessary and available
3798     if ($device->{usb3}) {
3799
3800         my $devicelist = vm_devices_list($vmid);
3801
3802         if (!$devicelist->{xhci}) {
3803             my $pciaddr = print_pci_addr("xhci");
3804             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3805         }
3806     }
3807     my $d = parse_usb_device($device->{host});
3808     $d->{usb3} = $device->{usb3};
3809
3810     # add the new one
3811     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3812 }
3813
3814 sub qemu_cpu_hotplug {
3815     my ($vmid, $conf, $vcpus) = @_;
3816
3817     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3818
3819     my $sockets = 1;
3820     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3821     $sockets = $conf->{sockets} if  $conf->{sockets};
3822     my $cores = $conf->{cores} || 1;
3823     my $maxcpus = $sockets * $cores;
3824
3825     $vcpus = $maxcpus if !$vcpus;
3826
3827     die "you can't add more vcpus than maxcpus\n"
3828         if $vcpus > $maxcpus;
3829
3830     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3831
3832     if ($vcpus < $currentvcpus) {
3833
3834         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3835
3836             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3837                 qemu_devicedel($vmid, "cpu$i");
3838                 my $retry = 0;
3839                 my $currentrunningvcpus = undef;
3840                 while (1) {
3841                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3842                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3843                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3844                     $retry++;
3845                     sleep 1;
3846                 }
3847                 #update conf after each succesfull cpu unplug
3848                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3849                 PVE::QemuConfig->write_config($vmid, $conf);
3850             }
3851         } else {
3852             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3853         }
3854
3855         return;
3856     }
3857
3858     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3859     die "vcpus in running vm does not match its configuration\n"
3860         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3861
3862     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3863
3864         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3865             my $cpustr = print_cpu_device($conf, $i);
3866             qemu_deviceadd($vmid, $cpustr);
3867
3868             my $retry = 0;
3869             my $currentrunningvcpus = undef;
3870             while (1) {
3871                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3872                 last if scalar(@{$currentrunningvcpus}) == $i;
3873                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3874                 sleep 1;
3875                 $retry++;
3876             }
3877             #update conf after each succesfull cpu hotplug
3878             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3879             PVE::QemuConfig->write_config($vmid, $conf);
3880         }
3881     } else {
3882
3883         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3884             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3885         }
3886     }
3887 }
3888
3889 sub qemu_block_set_io_throttle {
3890     my ($vmid, $deviceid,
3891         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3892         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3893         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3894         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3895
3896     return if !check_running($vmid) ;
3897
3898     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3899         bps => int($bps),
3900         bps_rd => int($bps_rd),
3901         bps_wr => int($bps_wr),
3902         iops => int($iops),
3903         iops_rd => int($iops_rd),
3904         iops_wr => int($iops_wr),
3905         bps_max => int($bps_max),
3906         bps_rd_max => int($bps_rd_max),
3907         bps_wr_max => int($bps_wr_max),
3908         iops_max => int($iops_max),
3909         iops_rd_max => int($iops_rd_max),
3910         iops_wr_max => int($iops_wr_max),
3911         bps_max_length => int($bps_max_length),
3912         bps_rd_max_length => int($bps_rd_max_length),
3913         bps_wr_max_length => int($bps_wr_max_length),
3914         iops_max_length => int($iops_max_length),
3915         iops_rd_max_length => int($iops_rd_max_length),
3916         iops_wr_max_length => int($iops_wr_max_length),
3917     );
3918
3919 }
3920
3921 # old code, only used to shutdown old VM after update
3922 sub __read_avail {
3923     my ($fh, $timeout) = @_;
3924
3925     my $sel = new IO::Select;
3926     $sel->add($fh);
3927
3928     my $res = '';
3929     my $buf;
3930
3931     my @ready;
3932     while (scalar (@ready = $sel->can_read($timeout))) {
3933         my $count;
3934         if ($count = $fh->sysread($buf, 8192)) {
3935             if ($buf =~ /^(.*)\(qemu\) $/s) {
3936                 $res .= $1;
3937                 last;
3938             } else {
3939                 $res .= $buf;
3940             }
3941         } else {
3942             if (!defined($count)) {
3943                 die "$!\n";
3944             }
3945             last;
3946         }
3947     }
3948
3949     die "monitor read timeout\n" if !scalar(@ready);
3950
3951     return $res;
3952 }
3953
3954 # old code, only used to shutdown old VM after update
3955 sub vm_monitor_command {
3956     my ($vmid, $cmdstr, $nocheck) = @_;
3957
3958     my $res;
3959
3960     eval {
3961         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3962
3963         my $sname = "${var_run_tmpdir}/$vmid.mon";
3964
3965         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3966             die "unable to connect to VM $vmid socket - $!\n";
3967
3968         my $timeout = 3;
3969
3970         # hack: migrate sometime blocks the monitor (when migrate_downtime
3971         # is set)
3972         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
3973             $timeout = 60*60; # 1 hour
3974         }
3975
3976         # read banner;
3977         my $data = __read_avail($sock, $timeout);
3978
3979         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
3980             die "got unexpected qemu monitor banner\n";
3981         }
3982
3983         my $sel = new IO::Select;
3984         $sel->add($sock);
3985
3986         if (!scalar(my @ready = $sel->can_write($timeout))) {
3987             die "monitor write error - timeout";
3988         }
3989
3990         my $fullcmd = "$cmdstr\r";
3991
3992         # syslog('info', "VM $vmid monitor command: $cmdstr");
3993
3994         my $b;
3995         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
3996             die "monitor write error - $!";
3997         }
3998
3999         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4000
4001         $timeout = 20;
4002
4003         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4004             $timeout = 60*60; # 1 hour
4005         } elsif ($cmdstr =~ m/^(eject|change)/) {
4006             $timeout = 60; # note: cdrom mount command is slow
4007         }
4008         if ($res = __read_avail($sock, $timeout)) {
4009
4010             my @lines = split("\r?\n", $res);
4011
4012             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4013
4014             $res = join("\n", @lines);
4015             $res .= "\n";
4016         }
4017     };
4018
4019     my $err = $@;
4020
4021     if ($err) {
4022         syslog("err", "VM $vmid monitor command failed - $err");
4023         die $err;
4024     }
4025
4026     return $res;
4027 }
4028
4029 sub qemu_block_resize {
4030     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4031
4032     my $running = check_running($vmid);
4033
4034     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4035
4036     return if !$running;
4037
4038     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4039
4040 }
4041
4042 sub qemu_volume_snapshot {
4043     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4044
4045     my $running = check_running($vmid);
4046
4047     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4048         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4049     } else {
4050         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4051     }
4052 }
4053
4054 sub qemu_volume_snapshot_delete {
4055     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4056
4057     my $running = check_running($vmid);
4058
4059     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4060         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4061     } else {
4062         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4063     }
4064 }
4065
4066 sub set_migration_caps {
4067     my ($vmid) = @_;
4068
4069     my $cap_ref = [];
4070
4071     my $enabled_cap = {
4072         "auto-converge" => 1,
4073         "xbzrle" => 1,
4074         "x-rdma-pin-all" => 0,
4075         "zero-blocks" => 0,
4076         "compress" => 0
4077     };
4078
4079     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4080
4081     for my $supported_capability (@$supported_capabilities) {
4082         push @$cap_ref, {
4083             capability => $supported_capability->{capability},
4084             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4085         };
4086     }
4087
4088     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4089 }
4090
4091 my $fast_plug_option = {
4092     'lock' => 1,
4093     'name' => 1,
4094     'onboot' => 1,
4095     'shares' => 1,
4096     'startup' => 1,
4097     'description' => 1,
4098     'protection' => 1,
4099     'vmstatestorage' => 1,
4100 };
4101
4102 # hotplug changes in [PENDING]
4103 # $selection hash can be used to only apply specified options, for
4104 # example: { cores => 1 } (only apply changed 'cores')
4105 # $errors ref is used to return error messages
4106 sub vmconfig_hotplug_pending {
4107     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4108
4109     my $defaults = load_defaults();
4110
4111     # commit values which do not have any impact on running VM first
4112     # Note: those option cannot raise errors, we we do not care about
4113     # $selection and always apply them.
4114
4115     my $add_error = sub {
4116         my ($opt, $msg) = @_;
4117         $errors->{$opt} = "hotplug problem - $msg";
4118     };
4119
4120     my $changes = 0;
4121     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4122         if ($fast_plug_option->{$opt}) {
4123             $conf->{$opt} = $conf->{pending}->{$opt};
4124             delete $conf->{pending}->{$opt};
4125             $changes = 1;
4126         }
4127     }
4128
4129     if ($changes) {
4130         PVE::QemuConfig->write_config($vmid, $conf);
4131         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4132     }
4133
4134     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4135
4136     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4137     while (my ($opt, $force) = each %$pending_delete_hash) {
4138         next if $selection && !$selection->{$opt};
4139         eval {
4140             if ($opt eq 'hotplug') {
4141                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4142             } elsif ($opt eq 'tablet') {
4143                 die "skip\n" if !$hotplug_features->{usb};
4144                 if ($defaults->{tablet}) {
4145                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4146                 } else {
4147                     vm_deviceunplug($vmid, $conf, $opt);
4148                 }
4149             } elsif ($opt =~ m/^usb\d+/) {
4150                 die "skip\n";
4151                 # since we cannot reliably hot unplug usb devices
4152                 # we are disabling it
4153                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4154                 vm_deviceunplug($vmid, $conf, $opt);
4155             } elsif ($opt eq 'vcpus') {
4156                 die "skip\n" if !$hotplug_features->{cpu};
4157                 qemu_cpu_hotplug($vmid, $conf, undef);
4158             } elsif ($opt eq 'balloon') {
4159                 # enable balloon device is not hotpluggable
4160                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4161             } elsif ($fast_plug_option->{$opt}) {
4162                 # do nothing
4163             } elsif ($opt =~ m/^net(\d+)$/) {
4164                 die "skip\n" if !$hotplug_features->{network};
4165                 vm_deviceunplug($vmid, $conf, $opt);
4166             } elsif (is_valid_drivename($opt)) {
4167                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4168                 vm_deviceunplug($vmid, $conf, $opt);
4169                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4170             } elsif ($opt =~ m/^memory$/) {
4171                 die "skip\n" if !$hotplug_features->{memory};
4172                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4173             } elsif ($opt eq 'cpuunits') {
4174                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4175             } elsif ($opt eq 'cpulimit') {
4176                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4177             } else {
4178                 die "skip\n";
4179             }
4180         };
4181         if (my $err = $@) {
4182             &$add_error($opt, $err) if $err ne "skip\n";
4183         } else {
4184             # save new config if hotplug was successful
4185             delete $conf->{$opt};
4186             vmconfig_undelete_pending_option($conf, $opt);
4187             PVE::QemuConfig->write_config($vmid, $conf);
4188             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4189         }
4190     }
4191
4192     foreach my $opt (keys %{$conf->{pending}}) {
4193         next if $selection && !$selection->{$opt};
4194         my $value = $conf->{pending}->{$opt};
4195         eval {
4196             if ($opt eq 'hotplug') {
4197                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4198             } elsif ($opt eq 'tablet') {
4199                 die "skip\n" if !$hotplug_features->{usb};
4200                 if ($value == 1) {
4201                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4202                 } elsif ($value == 0) {
4203                     vm_deviceunplug($vmid, $conf, $opt);
4204                 }
4205             } elsif ($opt =~ m/^usb\d+$/) {
4206                 die "skip\n";
4207                 # since we cannot reliably hot unplug usb devices
4208                 # we are disabling it
4209                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4210                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4211                 die "skip\n" if !$d;
4212                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4213             } elsif ($opt eq 'vcpus') {
4214                 die "skip\n" if !$hotplug_features->{cpu};
4215                 qemu_cpu_hotplug($vmid, $conf, $value);
4216             } elsif ($opt eq 'balloon') {
4217                 # enable/disable balloning device is not hotpluggable
4218                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4219                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4220                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4221
4222                 # allow manual ballooning if shares is set to zero
4223                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4224                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4225                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4226                 }
4227             } elsif ($opt =~ m/^net(\d+)$/) {
4228                 # some changes can be done without hotplug
4229                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4230                                     $vmid, $opt, $value);
4231             } elsif (is_valid_drivename($opt)) {
4232                 # some changes can be done without hotplug
4233                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4234                                      $vmid, $opt, $value, 1);
4235             } elsif ($opt =~ m/^memory$/) { #dimms
4236                 die "skip\n" if !$hotplug_features->{memory};
4237                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4238             } elsif ($opt eq 'cpuunits') {
4239                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4240             } elsif ($opt eq 'cpulimit') {
4241                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4242                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4243             } else {
4244                 die "skip\n";  # skip non-hot-pluggable options
4245             }
4246         };
4247         if (my $err = $@) {
4248             &$add_error($opt, $err) if $err ne "skip\n";
4249         } else {
4250             # save new config if hotplug was successful
4251             $conf->{$opt} = $value;
4252             delete $conf->{pending}->{$opt};
4253             PVE::QemuConfig->write_config($vmid, $conf);
4254             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4255         }
4256     }
4257 }
4258
4259 sub try_deallocate_drive {
4260     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4261
4262     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4263         my $volid = $drive->{file};
4264         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4265             my $sid = PVE::Storage::parse_volume_id($volid);
4266             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4267
4268             # check if the disk is really unused
4269             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4270                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4271             PVE::Storage::vdisk_free($storecfg, $volid);
4272             return 1;
4273         } else {
4274             # If vm is not owner of this disk remove from config
4275             return 1;
4276         }
4277     }
4278
4279     return undef;
4280 }
4281
4282 sub vmconfig_delete_or_detach_drive {
4283     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4284
4285     my $drive = parse_drive($opt, $conf->{$opt});
4286
4287     my $rpcenv = PVE::RPCEnvironment::get();
4288     my $authuser = $rpcenv->get_user();
4289
4290     if ($force) {
4291         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4292         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4293     } else {
4294         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4295     }
4296 }
4297
4298 sub vmconfig_apply_pending {
4299     my ($vmid, $conf, $storecfg) = @_;
4300
4301     # cold plug
4302
4303     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4304     while (my ($opt, $force) = each %$pending_delete_hash) {
4305         die "internal error" if $opt =~ m/^unused/;
4306         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4307         if (!defined($conf->{$opt})) {
4308             vmconfig_undelete_pending_option($conf, $opt);
4309             PVE::QemuConfig->write_config($vmid, $conf);
4310         } elsif (is_valid_drivename($opt)) {
4311             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4312             vmconfig_undelete_pending_option($conf, $opt);
4313             delete $conf->{$opt};
4314             PVE::QemuConfig->write_config($vmid, $conf);
4315         } else {
4316             vmconfig_undelete_pending_option($conf, $opt);
4317             delete $conf->{$opt};
4318             PVE::QemuConfig->write_config($vmid, $conf);
4319         }
4320     }
4321
4322     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4323
4324     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4325         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4326
4327         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4328             # skip if nothing changed
4329         } elsif (is_valid_drivename($opt)) {
4330             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4331                 if defined($conf->{$opt});
4332             $conf->{$opt} = $conf->{pending}->{$opt};
4333         } else {
4334             $conf->{$opt} = $conf->{pending}->{$opt};
4335         }
4336
4337         delete $conf->{pending}->{$opt};
4338         PVE::QemuConfig->write_config($vmid, $conf);
4339     }
4340 }
4341
4342 my $safe_num_ne = sub {
4343     my ($a, $b) = @_;
4344
4345     return 0 if !defined($a) && !defined($b);
4346     return 1 if !defined($a);
4347     return 1 if !defined($b);
4348
4349     return $a != $b;
4350 };
4351
4352 my $safe_string_ne = sub {
4353     my ($a, $b) = @_;
4354
4355     return 0 if !defined($a) && !defined($b);
4356     return 1 if !defined($a);
4357     return 1 if !defined($b);
4358
4359     return $a ne $b;
4360 };
4361
4362 sub vmconfig_update_net {
4363     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4364
4365     my $newnet = parse_net($value);
4366
4367     if ($conf->{$opt}) {
4368         my $oldnet = parse_net($conf->{$opt});
4369
4370         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4371             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4372             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4373             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4374
4375             # for non online change, we try to hot-unplug
4376             die "skip\n" if !$hotplug;
4377             vm_deviceunplug($vmid, $conf, $opt);
4378         } else {
4379
4380             die "internal error" if $opt !~ m/net(\d+)/;
4381             my $iface = "tap${vmid}i$1";
4382
4383             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4384                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4385                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4386                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4387                 PVE::Network::tap_unplug($iface);
4388                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4389             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4390                 # Rate can be applied on its own but any change above needs to
4391                 # include the rate in tap_plug since OVS resets everything.
4392                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4393             }
4394
4395             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4396                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4397             }
4398
4399             return 1;
4400         }
4401     }
4402
4403     if ($hotplug) {
4404         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4405     } else {
4406         die "skip\n";
4407     }
4408 }
4409
4410 sub vmconfig_update_disk {
4411     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4412
4413     # fixme: do we need force?
4414
4415     my $drive = parse_drive($opt, $value);
4416
4417     if ($conf->{$opt}) {
4418
4419         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4420
4421             my $media = $drive->{media} || 'disk';
4422             my $oldmedia = $old_drive->{media} || 'disk';
4423             die "unable to change media type\n" if $media ne $oldmedia;
4424
4425             if (!drive_is_cdrom($old_drive)) {
4426
4427                 if ($drive->{file} ne $old_drive->{file}) {
4428
4429                     die "skip\n" if !$hotplug;
4430
4431                     # unplug and register as unused
4432                     vm_deviceunplug($vmid, $conf, $opt);
4433                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4434
4435                 } else {
4436                     # update existing disk
4437
4438                     # skip non hotpluggable value
4439                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4440                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4441                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4442                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4443                         die "skip\n";
4444                     }
4445
4446                     # apply throttle
4447                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4448                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4449                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4450                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4451                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4452                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4453                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4454                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4455                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4456                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4457                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4458                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4459                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4460                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4461                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4462                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4463                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4464                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4465
4466                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4467                                                    ($drive->{mbps} || 0)*1024*1024,
4468                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4469                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4470                                                    $drive->{iops} || 0,
4471                                                    $drive->{iops_rd} || 0,
4472                                                    $drive->{iops_wr} || 0,
4473                                                    ($drive->{mbps_max} || 0)*1024*1024,
4474                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4475                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4476                                                    $drive->{iops_max} || 0,
4477                                                    $drive->{iops_rd_max} || 0,
4478                                                    $drive->{iops_wr_max} || 0,
4479                                                    $drive->{bps_max_length} || 1,
4480                                                    $drive->{bps_rd_max_length} || 1,
4481                                                    $drive->{bps_wr_max_length} || 1,
4482                                                    $drive->{iops_max_length} || 1,
4483                                                    $drive->{iops_rd_max_length} || 1,
4484                                                    $drive->{iops_wr_max_length} || 1);
4485
4486                     }
4487
4488                     return 1;
4489                 }
4490
4491             } else { # cdrom
4492
4493                 if ($drive->{file} eq 'none') {
4494                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4495                 } else {
4496                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4497                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4498                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4499                 }
4500
4501                 return 1;
4502             }
4503         }
4504     }
4505
4506     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4507     # hotplug new disks
4508     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4509     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4510 }
4511
4512 sub vm_start {
4513     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4514         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4515
4516     PVE::QemuConfig->lock_config($vmid, sub {
4517         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4518
4519         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4520
4521         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4522
4523         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4524
4525         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4526             vmconfig_apply_pending($vmid, $conf, $storecfg);
4527             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4528         }
4529
4530         my $defaults = load_defaults();
4531
4532         # set environment variable useful inside network script
4533         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4534
4535         my $local_volumes = {};
4536
4537         if ($targetstorage) {
4538             foreach_drive($conf, sub {
4539                 my ($ds, $drive) = @_;
4540
4541                 return if drive_is_cdrom($drive);
4542
4543                 my $volid = $drive->{file};
4544
4545                 return if !$volid;
4546
4547                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4548
4549                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4550                 return if $scfg->{shared};
4551                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4552             });
4553
4554             my $format = undef;
4555
4556             foreach my $opt (sort keys %$local_volumes) {
4557
4558                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4559                 my $drive = parse_drive($opt, $conf->{$opt});
4560
4561                 #if remote storage is specified, use default format
4562                 if ($targetstorage && $targetstorage ne "1") {
4563                     $storeid = $targetstorage;
4564                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4565                     $format = $defFormat;
4566                 } else {
4567                     #else we use same format than original
4568                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4569                     $format = qemu_img_format($scfg, $volid);
4570                 }
4571
4572                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4573                 my $newdrive = $drive;
4574                 $newdrive->{format} = $format;
4575                 $newdrive->{file} = $newvolid;
4576                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4577                 $local_volumes->{$opt} = $drivestr;
4578                 #pass drive to conf for command line
4579                 $conf->{$opt} = $drivestr;
4580             }
4581         }
4582
4583         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4584
4585         my $migrate_port = 0;
4586         my $migrate_uri;
4587         if ($statefile) {
4588             if ($statefile eq 'tcp') {
4589                 my $localip = "localhost";
4590                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4591                 my $nodename = PVE::INotify::nodename();
4592
4593                 if (!defined($migration_type)) {
4594                     if (defined($datacenterconf->{migration}->{type})) {
4595                         $migration_type = $datacenterconf->{migration}->{type};
4596                     } else {
4597                         $migration_type = 'secure';
4598                     }
4599                 }
4600
4601                 if ($migration_type eq 'insecure') {
4602                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4603                     if ($migrate_network_addr) {
4604                         $localip = $migrate_network_addr;
4605                     } else {
4606                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4607                     }
4608
4609                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4610                 }
4611
4612                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4613                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4614                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4615                 push @$cmd, '-incoming', $migrate_uri;
4616                 push @$cmd, '-S';
4617
4618             } elsif ($statefile eq 'unix') {
4619                 # should be default for secure migrations as a ssh TCP forward
4620                 # tunnel is not deterministic reliable ready and fails regurarly
4621                 # to set up in time, so use UNIX socket forwards
4622                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4623                 unlink $socket_addr;
4624
4625                 $migrate_uri = "unix:$socket_addr";
4626
4627                 push @$cmd, '-incoming', $migrate_uri;
4628                 push @$cmd, '-S';
4629
4630             } else {
4631                 push @$cmd, '-loadstate', $statefile;
4632             }
4633         } elsif ($paused) {
4634             push @$cmd, '-S';
4635         }
4636
4637         # host pci devices
4638         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4639           my $d = parse_hostpci($conf->{"hostpci$i"});
4640           next if !$d;
4641           my $pcidevices = $d->{pciid};
4642           foreach my $pcidevice (@$pcidevices) {
4643                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4644
4645                 my $info = pci_device_info("0000:$pciid");
4646                 die "IOMMU not present\n" if !check_iommu_support();
4647                 die "no pci device info for device '$pciid'\n" if !$info;
4648                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4649                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4650           }
4651         }
4652
4653         PVE::Storage::activate_volumes($storecfg, $vollist);
4654
4655         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4656             my $cmd = [];
4657             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4658             eval  { run_command($cmd); };
4659         }
4660
4661         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4662                                                   : $defaults->{cpuunits};
4663
4664         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4665
4666         my %properties = (
4667             Slice => 'qemu.slice',
4668             KillMode => 'none',
4669             CPUShares => $cpuunits
4670         );
4671
4672         if (my $cpulimit = $conf->{cpulimit}) {
4673             $properties{CPUQuota} = int($cpulimit * 100);
4674         }
4675         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4676
4677         if ($conf->{hugepages}) {
4678
4679             my $code = sub {
4680                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4681                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4682
4683                 PVE::QemuServer::Memory::hugepages_mount();
4684                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4685
4686                 eval  {
4687                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4688                     run_command($cmd, %run_params);
4689                 };
4690
4691                 if (my $err = $@) {
4692                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4693                     die $err;
4694                 }
4695
4696                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4697             };
4698             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4699
4700         } else {
4701             eval  {
4702                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4703                 run_command($cmd, %run_params);
4704             };
4705         }
4706
4707         if (my $err = $@) {
4708             # deactivate volumes if start fails
4709             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4710             die "start failed: $err";
4711         }
4712
4713         print "migration listens on $migrate_uri\n" if $migrate_uri;
4714
4715         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4716             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4717             warn $@ if $@;
4718         }
4719
4720         #start nbd server for storage migration
4721         if ($targetstorage) {
4722             my $nodename = PVE::INotify::nodename();
4723             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4724             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4725             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4726             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4727
4728             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4729
4730             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4731
4732             foreach my $opt (sort keys %$local_volumes) {
4733                 my $volid = $local_volumes->{$opt};
4734                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4735                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4736                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4737             }
4738         }
4739
4740         if ($migratedfrom) {
4741             eval {
4742                 set_migration_caps($vmid);
4743             };
4744             warn $@ if $@;
4745
4746             if ($spice_port) {
4747                 print "spice listens on port $spice_port\n";
4748                 if ($spice_ticket) {
4749                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4750                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4751                 }
4752             }
4753
4754         } else {
4755             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4756                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4757                     if $conf->{balloon};
4758             }
4759
4760             foreach my $opt (keys %$conf) {
4761                 next if $opt !~  m/^net\d+$/;
4762                 my $nicconf = parse_net($conf->{$opt});
4763                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4764             }
4765         }
4766
4767         vm_mon_cmd_nocheck($vmid, 'qom-set',
4768                     path => "machine/peripheral/balloon0",
4769                     property => "guest-stats-polling-interval",
4770                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4771
4772     });
4773 }
4774
4775 sub vm_mon_cmd {
4776     my ($vmid, $execute, %params) = @_;
4777
4778     my $cmd = { execute => $execute, arguments => \%params };
4779     vm_qmp_command($vmid, $cmd);
4780 }
4781
4782 sub vm_mon_cmd_nocheck {
4783     my ($vmid, $execute, %params) = @_;
4784
4785     my $cmd = { execute => $execute, arguments => \%params };
4786     vm_qmp_command($vmid, $cmd, 1);
4787 }
4788
4789 sub vm_qmp_command {
4790     my ($vmid, $cmd, $nocheck) = @_;
4791
4792     my $res;
4793
4794     my $timeout;
4795     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4796         $timeout = $cmd->{arguments}->{timeout};
4797         delete $cmd->{arguments}->{timeout};
4798     }
4799
4800     eval {
4801         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4802         my $sname = qmp_socket($vmid);
4803         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4804             my $qmpclient = PVE::QMPClient->new();
4805
4806             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4807         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4808             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4809                 if scalar(%{$cmd->{arguments}});
4810             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4811         } else {
4812             die "unable to open monitor socket\n";
4813         }
4814     };
4815     if (my $err = $@) {
4816         syslog("err", "VM $vmid qmp command failed - $err");
4817         die $err;
4818     }
4819
4820     return $res;
4821 }
4822
4823 sub vm_human_monitor_command {
4824     my ($vmid, $cmdline) = @_;
4825
4826     my $res;
4827
4828     my $cmd = {
4829         execute => 'human-monitor-command',
4830         arguments => { 'command-line' => $cmdline},
4831     };
4832
4833     return vm_qmp_command($vmid, $cmd);
4834 }
4835
4836 sub vm_commandline {
4837     my ($storecfg, $vmid) = @_;
4838
4839     my $conf = PVE::QemuConfig->load_config($vmid);
4840
4841     my $defaults = load_defaults();
4842
4843     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4844
4845     return PVE::Tools::cmd2string($cmd);
4846 }
4847
4848 sub vm_reset {
4849     my ($vmid, $skiplock) = @_;
4850
4851     PVE::QemuConfig->lock_config($vmid, sub {
4852
4853         my $conf = PVE::QemuConfig->load_config($vmid);
4854
4855         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4856
4857         vm_mon_cmd($vmid, "system_reset");
4858     });
4859 }
4860
4861 sub get_vm_volumes {
4862     my ($conf) = @_;
4863
4864     my $vollist = [];
4865     foreach_volid($conf, sub {
4866         my ($volid, $attr) = @_;
4867
4868         return if $volid =~ m|^/|;
4869
4870         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4871         return if !$sid;
4872
4873         push @$vollist, $volid;
4874     });
4875
4876     return $vollist;
4877 }
4878
4879 sub vm_stop_cleanup {
4880     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4881
4882     eval {
4883
4884         if (!$keepActive) {
4885             my $vollist = get_vm_volumes($conf);
4886             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4887         }
4888
4889         foreach my $ext (qw(mon qmp pid vnc qga)) {
4890             unlink "/var/run/qemu-server/${vmid}.$ext";
4891         }
4892
4893         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4894     };
4895     warn $@ if $@; # avoid errors - just warn
4896 }
4897
4898 # Note: use $nockeck to skip tests if VM configuration file exists.
4899 # We need that when migration VMs to other nodes (files already moved)
4900 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4901 sub vm_stop {
4902     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4903
4904     $force = 1 if !defined($force) && !$shutdown;
4905
4906     if ($migratedfrom){
4907         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4908         kill 15, $pid if $pid;
4909         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4910         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4911         return;
4912     }
4913
4914     PVE::QemuConfig->lock_config($vmid, sub {
4915
4916         my $pid = check_running($vmid, $nocheck);
4917         return if !$pid;
4918
4919         my $conf;
4920         if (!$nocheck) {
4921             $conf = PVE::QemuConfig->load_config($vmid);
4922             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4923             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4924                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4925                 $timeout = $opts->{down} if $opts->{down};
4926             }
4927         }
4928
4929         $timeout = 60 if !defined($timeout);
4930
4931         eval {
4932             if ($shutdown) {
4933                 if (defined($conf) && $conf->{agent}) {
4934                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4935                 } else {
4936                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4937                 }
4938             } else {
4939                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4940             }
4941         };
4942         my $err = $@;
4943
4944         if (!$err) {
4945             my $count = 0;
4946             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4947                 $count++;
4948                 sleep 1;
4949             }
4950
4951             if ($count >= $timeout) {
4952                 if ($force) {
4953                     warn "VM still running - terminating now with SIGTERM\n";
4954                     kill 15, $pid;
4955                 } else {
4956                     die "VM quit/powerdown failed - got timeout\n";
4957                 }
4958             } else {
4959                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4960                 return;
4961             }
4962         } else {
4963             if ($force) {
4964                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4965                 kill 15, $pid;
4966             } else {
4967                 die "VM quit/powerdown failed\n";
4968             }
4969         }
4970
4971         # wait again
4972         $timeout = 10;
4973
4974         my $count = 0;
4975         while (($count < $timeout) && check_running($vmid, $nocheck)) {
4976             $count++;
4977             sleep 1;
4978         }
4979
4980         if ($count >= $timeout) {
4981             warn "VM still running - terminating now with SIGKILL\n";
4982             kill 9, $pid;
4983             sleep 1;
4984         }
4985
4986         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4987    });
4988 }
4989
4990 sub vm_suspend {
4991     my ($vmid, $skiplock) = @_;
4992
4993     PVE::QemuConfig->lock_config($vmid, sub {
4994
4995         my $conf = PVE::QemuConfig->load_config($vmid);
4996
4997         PVE::QemuConfig->check_lock($conf)
4998             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
4999
5000         vm_mon_cmd($vmid, "stop");
5001     });
5002 }
5003
5004 sub vm_resume {
5005     my ($vmid, $skiplock, $nocheck) = @_;
5006
5007     PVE::QemuConfig->lock_config($vmid, sub {
5008
5009         if (!$nocheck) {
5010
5011             my $conf = PVE::QemuConfig->load_config($vmid);
5012
5013             PVE::QemuConfig->check_lock($conf)
5014                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5015
5016             vm_mon_cmd($vmid, "cont");
5017
5018         } else {
5019             vm_mon_cmd_nocheck($vmid, "cont");
5020         }
5021     });
5022 }
5023
5024 sub vm_sendkey {
5025     my ($vmid, $skiplock, $key) = @_;
5026
5027     PVE::QemuConfig->lock_config($vmid, sub {
5028
5029         my $conf = PVE::QemuConfig->load_config($vmid);
5030
5031         # there is no qmp command, so we use the human monitor command
5032         vm_human_monitor_command($vmid, "sendkey $key");
5033     });
5034 }
5035
5036 sub vm_destroy {
5037     my ($storecfg, $vmid, $skiplock) = @_;
5038
5039     PVE::QemuConfig->lock_config($vmid, sub {
5040
5041         my $conf = PVE::QemuConfig->load_config($vmid);
5042
5043         if (!check_running($vmid)) {
5044             destroy_vm($storecfg, $vmid, undef, $skiplock);
5045         } else {
5046             die "VM $vmid is running - destroy failed\n";
5047         }
5048     });
5049 }
5050
5051 # pci helpers
5052
5053 sub file_write {
5054     my ($filename, $buf) = @_;
5055
5056     my $fh = IO::File->new($filename, "w");
5057     return undef if !$fh;
5058
5059     my $res = print $fh $buf;
5060
5061     $fh->close();
5062
5063     return $res;
5064 }
5065
5066 sub pci_device_info {
5067     my ($name) = @_;
5068
5069     my $res;
5070
5071     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5072     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5073
5074     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5075     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5076
5077     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5078     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5079
5080     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5081     return undef if !defined($product) || $product !~ s/^0x//;
5082
5083     $res = {
5084         name => $name,
5085         vendor => $vendor,
5086         product => $product,
5087         domain => $domain,
5088         bus => $bus,
5089         slot => $slot,
5090         func => $func,
5091         irq => $irq,
5092         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5093     };
5094
5095     return $res;
5096 }
5097
5098 sub pci_dev_reset {
5099     my ($dev) = @_;
5100
5101     my $name = $dev->{name};
5102
5103     my $fn = "$pcisysfs/devices/$name/reset";
5104
5105     return file_write($fn, "1");
5106 }
5107
5108 sub pci_dev_bind_to_vfio {
5109     my ($dev) = @_;
5110
5111     my $name = $dev->{name};
5112
5113     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5114
5115     if (!-d $vfio_basedir) {
5116         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5117     }
5118     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5119
5120     my $testdir = "$vfio_basedir/$name";
5121     return 1 if -d $testdir;
5122
5123     my $data = "$dev->{vendor} $dev->{product}";
5124     return undef if !file_write("$vfio_basedir/new_id", $data);
5125
5126     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5127     if (!file_write($fn, $name)) {
5128         return undef if -f $fn;
5129     }
5130
5131     $fn = "$vfio_basedir/bind";
5132     if (! -d $testdir) {
5133         return undef if !file_write($fn, $name);
5134     }
5135
5136     return -d $testdir;
5137 }
5138
5139 sub pci_dev_group_bind_to_vfio {
5140     my ($pciid) = @_;
5141
5142     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5143
5144     if (!-d $vfio_basedir) {
5145         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5146     }
5147     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5148
5149     # get IOMMU group devices
5150     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5151       my @devs = grep /^0000:/, readdir($D);
5152     closedir($D);
5153
5154     foreach my $pciid (@devs) {
5155         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5156
5157         # pci bridges, switches or root ports are not supported
5158         # they have a pci_bus subdirectory so skip them
5159         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5160
5161         my $info = pci_device_info($1);
5162         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5163     }
5164
5165     return 1;
5166 }
5167
5168 # vzdump restore implementaion
5169
5170 sub tar_archive_read_firstfile {
5171     my $archive = shift;
5172
5173     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5174
5175     # try to detect archive type first
5176     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5177         die "unable to open file '$archive'\n";
5178     my $firstfile = <$fh>;
5179     kill 15, $pid;
5180     close $fh;
5181
5182     die "ERROR: archive contaions no data\n" if !$firstfile;
5183     chomp $firstfile;
5184
5185     return $firstfile;
5186 }
5187
5188 sub tar_restore_cleanup {
5189     my ($storecfg, $statfile) = @_;
5190
5191     print STDERR "starting cleanup\n";
5192
5193     if (my $fd = IO::File->new($statfile, "r")) {
5194         while (defined(my $line = <$fd>)) {
5195             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5196                 my $volid = $2;
5197                 eval {
5198                     if ($volid =~ m|^/|) {
5199                         unlink $volid || die 'unlink failed\n';
5200                     } else {
5201                         PVE::Storage::vdisk_free($storecfg, $volid);
5202                     }
5203                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5204                 };
5205                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5206             } else {
5207                 print STDERR "unable to parse line in statfile - $line";
5208             }
5209         }
5210         $fd->close();
5211     }
5212 }
5213
5214 sub restore_archive {
5215     my ($archive, $vmid, $user, $opts) = @_;
5216
5217     my $format = $opts->{format};
5218     my $comp;
5219
5220     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5221         $format = 'tar' if !$format;
5222         $comp = 'gzip';
5223     } elsif ($archive =~ m/\.tar$/) {
5224         $format = 'tar' if !$format;
5225     } elsif ($archive =~ m/.tar.lzo$/) {
5226         $format = 'tar' if !$format;
5227         $comp = 'lzop';
5228     } elsif ($archive =~ m/\.vma$/) {
5229         $format = 'vma' if !$format;
5230     } elsif ($archive =~ m/\.vma\.gz$/) {
5231         $format = 'vma' if !$format;
5232         $comp = 'gzip';
5233     } elsif ($archive =~ m/\.vma\.lzo$/) {
5234         $format = 'vma' if !$format;
5235         $comp = 'lzop';
5236     } else {
5237         $format = 'vma' if !$format; # default
5238     }
5239
5240     # try to detect archive format
5241     if ($format eq 'tar') {
5242         return restore_tar_archive($archive, $vmid, $user, $opts);
5243     } else {
5244         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5245     }
5246 }
5247
5248 sub restore_update_config_line {
5249     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5250
5251     return if $line =~ m/^\#qmdump\#/;
5252     return if $line =~ m/^\#vzdump\#/;
5253     return if $line =~ m/^lock:/;
5254     return if $line =~ m/^unused\d+:/;
5255     return if $line =~ m/^parent:/;
5256     return if $line =~ m/^template:/; # restored VM is never a template
5257
5258     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5259     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5260         # try to convert old 1.X settings
5261         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5262         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5263             my ($model, $macaddr) = split(/\=/, $devconfig);
5264             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5265             my $net = {
5266                 model => $model,
5267                 bridge => "vmbr$ind",
5268                 macaddr => $macaddr,
5269             };
5270             my $netstr = print_net($net);
5271
5272             print $outfd "net$cookie->{netcount}: $netstr\n";
5273             $cookie->{netcount}++;
5274         }
5275     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5276         my ($id, $netstr) = ($1, $2);
5277         my $net = parse_net($netstr);
5278         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5279         $netstr = print_net($net);
5280         print $outfd "$id: $netstr\n";
5281     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5282         my $virtdev = $1;
5283         my $value = $3;
5284         my $di = parse_drive($virtdev, $value);
5285         if (defined($di->{backup}) && !$di->{backup}) {
5286             print $outfd "#$line";
5287         } elsif ($map->{$virtdev}) {
5288             delete $di->{format}; # format can change on restore
5289             $di->{file} = $map->{$virtdev};
5290             $value = print_drive($vmid, $di);
5291             print $outfd "$virtdev: $value\n";
5292         } else {
5293             print $outfd $line;
5294         }
5295     } else {
5296         print $outfd $line;
5297     }
5298 }
5299
5300 sub scan_volids {
5301     my ($cfg, $vmid) = @_;
5302
5303     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5304
5305     my $volid_hash = {};
5306     foreach my $storeid (keys %$info) {
5307         foreach my $item (@{$info->{$storeid}}) {
5308             next if !($item->{volid} && $item->{size});
5309             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5310             $volid_hash->{$item->{volid}} = $item;
5311         }
5312     }
5313
5314     return $volid_hash;
5315 }
5316
5317 sub is_volume_in_use {
5318     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5319
5320     my $path = PVE::Storage::path($storecfg, $volid);
5321
5322     my $scan_config = sub {
5323         my ($cref, $snapname) = @_;
5324
5325         foreach my $key (keys %$cref) {
5326             my $value = $cref->{$key};
5327             if (is_valid_drivename($key)) {
5328                 next if $skip_drive && $key eq $skip_drive;
5329                 my $drive = parse_drive($key, $value);
5330                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5331                 return 1 if $volid eq $drive->{file};
5332                 if ($drive->{file} =~ m!^/!) {
5333                     return 1 if $drive->{file} eq $path;
5334                 } else {
5335                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5336                     next if !$storeid;
5337                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5338                     next if !$scfg;
5339                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5340                 }
5341             }
5342         }
5343
5344         return 0;
5345     };
5346
5347     return 1 if &$scan_config($conf);
5348
5349     undef $skip_drive;
5350
5351     foreach my $snapname (keys %{$conf->{snapshots}}) {
5352         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5353     }
5354
5355     return 0;
5356 }
5357
5358 sub update_disksize {
5359     my ($vmid, $conf, $volid_hash) = @_;
5360
5361     my $changes;
5362
5363     my $used = {};
5364
5365     # Note: it is allowed to define multiple storages with same path (alias), so
5366     # we need to check both 'volid' and real 'path' (two different volid can point
5367     # to the same path).
5368
5369     my $usedpath = {};
5370
5371     # update size info
5372     foreach my $opt (keys %$conf) {
5373         if (is_valid_drivename($opt)) {
5374             my $drive = parse_drive($opt, $conf->{$opt});
5375             my $volid = $drive->{file};
5376             next if !$volid;
5377
5378             $used->{$volid} = 1;
5379             if ($volid_hash->{$volid} &&
5380                 (my $path = $volid_hash->{$volid}->{path})) {
5381                 $usedpath->{$path} = 1;
5382             }
5383
5384             next if drive_is_cdrom($drive);
5385             next if !$volid_hash->{$volid};
5386
5387             $drive->{size} = $volid_hash->{$volid}->{size};
5388             my $new = print_drive($vmid, $drive);
5389             if ($new ne $conf->{$opt}) {
5390                 $changes = 1;
5391                 $conf->{$opt} = $new;
5392             }
5393         }
5394     }
5395
5396     # remove 'unusedX' entry if volume is used
5397     foreach my $opt (keys %$conf) {
5398         next if $opt !~ m/^unused\d+$/;
5399         my $volid = $conf->{$opt};
5400         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5401         if ($used->{$volid} || ($path && $usedpath->{$path})) {
5402             $changes = 1;
5403             delete $conf->{$opt};
5404         }
5405     }
5406
5407     foreach my $volid (sort keys %$volid_hash) {
5408         next if $volid =~ m/vm-$vmid-state-/;
5409         next if $used->{$volid};
5410         my $path = $volid_hash->{$volid}->{path};
5411         next if !$path; # just to be sure
5412         next if $usedpath->{$path};
5413         $changes = 1;
5414         PVE::QemuConfig->add_unused_volume($conf, $volid);
5415         $usedpath->{$path} = 1; # avoid to add more than once (aliases)
5416     }
5417
5418     return $changes;
5419 }
5420
5421 sub rescan {
5422     my ($vmid, $nolock) = @_;
5423
5424     my $cfg = PVE::Storage::config();
5425
5426     my $volid_hash = scan_volids($cfg, $vmid);
5427
5428     my $updatefn =  sub {
5429         my ($vmid) = @_;
5430
5431         my $conf = PVE::QemuConfig->load_config($vmid);
5432
5433         PVE::QemuConfig->check_lock($conf);
5434
5435         my $vm_volids = {};
5436         foreach my $volid (keys %$volid_hash) {
5437             my $info = $volid_hash->{$volid};
5438             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5439         }
5440
5441         my $changes = update_disksize($vmid, $conf, $vm_volids);
5442
5443         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5444     };
5445
5446     if (defined($vmid)) {
5447         if ($nolock) {
5448             &$updatefn($vmid);
5449         } else {
5450             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5451         }
5452     } else {
5453         my $vmlist = config_list();
5454         foreach my $vmid (keys %$vmlist) {
5455             if ($nolock) {
5456                 &$updatefn($vmid);
5457             } else {
5458                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5459             }
5460         }
5461     }
5462 }
5463
5464 sub restore_vma_archive {
5465     my ($archive, $vmid, $user, $opts, $comp) = @_;
5466
5467     my $input = $archive eq '-' ? "<&STDIN" : undef;
5468     my $readfrom = $archive;
5469
5470     my $uncomp = '';
5471     if ($comp) {
5472         $readfrom = '-';
5473         my $qarchive = PVE::Tools::shellquote($archive);
5474         if ($comp eq 'gzip') {
5475             $uncomp = "zcat $qarchive|";
5476         } elsif ($comp eq 'lzop') {
5477             $uncomp = "lzop -d -c $qarchive|";
5478         } else {
5479             die "unknown compression method '$comp'\n";
5480         }
5481
5482     }
5483
5484     my $tmpdir = "/var/tmp/vzdumptmp$$";
5485     rmtree $tmpdir;
5486
5487     # disable interrupts (always do cleanups)
5488     local $SIG{INT} =
5489         local $SIG{TERM} =
5490         local $SIG{QUIT} =
5491         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5492
5493     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5494     POSIX::mkfifo($mapfifo, 0600);
5495     my $fifofh;
5496
5497     my $openfifo = sub {
5498         open($fifofh, '>', $mapfifo) || die $!;
5499     };
5500
5501     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5502
5503     my $oldtimeout;
5504     my $timeout = 5;
5505
5506     my $devinfo = {};
5507
5508     my $rpcenv = PVE::RPCEnvironment::get();
5509
5510     my $conffile = PVE::QemuConfig->config_file($vmid);
5511     my $tmpfn = "$conffile.$$.tmp";
5512
5513     # Note: $oldconf is undef if VM does not exists
5514     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5515     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5516
5517     my $print_devmap = sub {
5518         my $virtdev_hash = {};
5519
5520         my $cfgfn = "$tmpdir/qemu-server.conf";
5521
5522         # we can read the config - that is already extracted
5523         my $fh = IO::File->new($cfgfn, "r") ||
5524             "unable to read qemu-server.conf - $!\n";
5525
5526         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5527         if (-f $fwcfgfn) {
5528             my $pve_firewall_dir = '/etc/pve/firewall';
5529             mkdir $pve_firewall_dir; # make sure the dir exists
5530             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5531         }
5532
5533         while (defined(my $line = <$fh>)) {
5534             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5535                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5536                 die "archive does not contain data for drive '$virtdev'\n"
5537                     if !$devinfo->{$devname};
5538                 if (defined($opts->{storage})) {
5539                     $storeid = $opts->{storage} || 'local';
5540                 } elsif (!$storeid) {
5541                     $storeid = 'local';
5542                 }
5543                 $format = 'raw' if !$format;
5544                 $devinfo->{$devname}->{devname} = $devname;
5545                 $devinfo->{$devname}->{virtdev} = $virtdev;
5546                 $devinfo->{$devname}->{format} = $format;
5547                 $devinfo->{$devname}->{storeid} = $storeid;
5548
5549                 # check permission on storage
5550                 my $pool = $opts->{pool}; # todo: do we need that?
5551                 if ($user ne 'root@pam') {
5552                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5553                 }
5554
5555                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5556             }
5557         }
5558
5559         foreach my $devname (keys %$devinfo) {
5560             die "found no device mapping information for device '$devname'\n"
5561                 if !$devinfo->{$devname}->{virtdev};
5562         }
5563
5564         my $cfg = PVE::Storage::config();
5565
5566         # create empty/temp config
5567         if ($oldconf) {
5568             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5569             foreach_drive($oldconf, sub {
5570                 my ($ds, $drive) = @_;
5571
5572                 return if drive_is_cdrom($drive);
5573
5574                 my $volid = $drive->{file};
5575
5576                 return if !$volid || $volid =~ m|^/|;
5577
5578                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5579                 return if !$path || !$owner || ($owner != $vmid);
5580
5581                 # Note: only delete disk we want to restore
5582                 # other volumes will become unused
5583                 if ($virtdev_hash->{$ds}) {
5584                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5585                     if (my $err = $@) {
5586                         warn $err;
5587                     }
5588                 }
5589             });
5590
5591             # delete vmstate files
5592             # since after the restore we have no snapshots anymore
5593             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5594                 my $snap = $oldconf->{snapshots}->{$snapname};
5595                 if ($snap->{vmstate}) {
5596                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5597                     if (my $err = $@) {
5598                         warn $err;
5599                     }
5600                 }
5601             }
5602         }
5603
5604         my $map = {};
5605         foreach my $virtdev (sort keys %$virtdev_hash) {
5606             my $d = $virtdev_hash->{$virtdev};
5607             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5608             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5609
5610             # test if requested format is supported
5611             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5612             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5613             $d->{format} = $defFormat if !$supported;
5614
5615             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5616                                                   $d->{format}, undef, $alloc_size);
5617             print STDERR "new volume ID is '$volid'\n";
5618             $d->{volid} = $volid;
5619             my $path = PVE::Storage::path($cfg, $volid);
5620
5621             PVE::Storage::activate_volumes($cfg,[$volid]);
5622
5623             my $write_zeros = 1;
5624             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5625                 $write_zeros = 0;
5626             }
5627
5628             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5629
5630             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5631             $map->{$virtdev} = $volid;
5632         }
5633
5634         $fh->seek(0, 0) || die "seek failed - $!\n";
5635
5636         my $outfd = new IO::File ($tmpfn, "w") ||
5637             die "unable to write config for VM $vmid\n";
5638
5639         my $cookie = { netcount => 0 };
5640         while (defined(my $line = <$fh>)) {
5641             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5642         }
5643
5644         $fh->close();
5645         $outfd->close();
5646     };
5647
5648     eval {
5649         # enable interrupts
5650         local $SIG{INT} =
5651             local $SIG{TERM} =
5652             local $SIG{QUIT} =
5653             local $SIG{HUP} =
5654             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5655         local $SIG{ALRM} = sub { die "got timeout\n"; };
5656
5657         $oldtimeout = alarm($timeout);
5658
5659         my $parser = sub {
5660             my $line = shift;
5661
5662             print "$line\n";
5663
5664             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5665                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5666                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5667             } elsif ($line =~ m/^CTIME: /) {
5668                 # we correctly received the vma config, so we can disable
5669                 # the timeout now for disk allocation (set to 10 minutes, so
5670                 # that we always timeout if something goes wrong)
5671                 alarm(600);
5672                 &$print_devmap();
5673                 print $fifofh "done\n";
5674                 my $tmp = $oldtimeout || 0;
5675                 $oldtimeout = undef;
5676                 alarm($tmp);
5677                 close($fifofh);
5678             }
5679         };
5680
5681         print "restore vma archive: $cmd\n";
5682         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5683     };
5684     my $err = $@;
5685
5686     alarm($oldtimeout) if $oldtimeout;
5687
5688     my $vollist = [];
5689     foreach my $devname (keys %$devinfo) {
5690         my $volid = $devinfo->{$devname}->{volid};
5691         push @$vollist, $volid if $volid;
5692     }
5693
5694     my $cfg = PVE::Storage::config();
5695     PVE::Storage::deactivate_volumes($cfg, $vollist);
5696
5697     unlink $mapfifo;
5698
5699     if ($err) {
5700         rmtree $tmpdir;
5701         unlink $tmpfn;
5702
5703         foreach my $devname (keys %$devinfo) {
5704             my $volid = $devinfo->{$devname}->{volid};
5705             next if !$volid;
5706             eval {
5707                 if ($volid =~ m|^/|) {
5708                     unlink $volid || die 'unlink failed\n';
5709                 } else {
5710                     PVE::Storage::vdisk_free($cfg, $volid);
5711                 }
5712                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5713             };
5714             print STDERR "unable to cleanup '$volid' - $@" if $@;
5715         }
5716         die $err;
5717     }
5718
5719     rmtree $tmpdir;
5720
5721     rename($tmpfn, $conffile) ||
5722         die "unable to commit configuration file '$conffile'\n";
5723
5724     PVE::Cluster::cfs_update(); # make sure we read new file
5725
5726     eval { rescan($vmid, 1); };
5727     warn $@ if $@;
5728 }
5729
5730 sub restore_tar_archive {
5731     my ($archive, $vmid, $user, $opts) = @_;
5732
5733     if ($archive ne '-') {
5734         my $firstfile = tar_archive_read_firstfile($archive);
5735         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5736             if $firstfile ne 'qemu-server.conf';
5737     }
5738
5739     my $storecfg = PVE::Storage::config();
5740
5741     # destroy existing data - keep empty config
5742     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5743     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5744
5745     my $tocmd = "/usr/lib/qemu-server/qmextract";
5746
5747     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5748     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5749     $tocmd .= ' --prealloc' if $opts->{prealloc};
5750     $tocmd .= ' --info' if $opts->{info};
5751
5752     # tar option "xf" does not autodetect compression when read from STDIN,
5753     # so we pipe to zcat
5754     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5755         PVE::Tools::shellquote("--to-command=$tocmd");
5756
5757     my $tmpdir = "/var/tmp/vzdumptmp$$";
5758     mkpath $tmpdir;
5759
5760     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5761     local $ENV{VZDUMP_VMID} = $vmid;
5762     local $ENV{VZDUMP_USER} = $user;
5763
5764     my $conffile = PVE::QemuConfig->config_file($vmid);
5765     my $tmpfn = "$conffile.$$.tmp";
5766
5767     # disable interrupts (always do cleanups)
5768     local $SIG{INT} =
5769         local $SIG{TERM} =
5770         local $SIG{QUIT} =
5771         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5772
5773     eval {
5774         # enable interrupts
5775         local $SIG{INT} =
5776             local $SIG{TERM} =
5777             local $SIG{QUIT} =
5778             local $SIG{HUP} =
5779             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5780
5781         if ($archive eq '-') {
5782             print "extracting archive from STDIN\n";
5783             run_command($cmd, input => "<&STDIN");
5784         } else {
5785             print "extracting archive '$archive'\n";
5786             run_command($cmd);
5787         }
5788
5789         return if $opts->{info};
5790
5791         # read new mapping
5792         my $map = {};
5793         my $statfile = "$tmpdir/qmrestore.stat";
5794         if (my $fd = IO::File->new($statfile, "r")) {
5795             while (defined (my $line = <$fd>)) {
5796                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5797                     $map->{$1} = $2 if $1;
5798                 } else {
5799                     print STDERR "unable to parse line in statfile - $line\n";
5800                 }
5801             }
5802             $fd->close();
5803         }
5804
5805         my $confsrc = "$tmpdir/qemu-server.conf";
5806
5807         my $srcfd = new IO::File($confsrc, "r") ||
5808             die "unable to open file '$confsrc'\n";
5809
5810         my $outfd = new IO::File ($tmpfn, "w") ||
5811             die "unable to write config for VM $vmid\n";
5812
5813         my $cookie = { netcount => 0 };
5814         while (defined (my $line = <$srcfd>)) {
5815             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5816         }
5817
5818         $srcfd->close();
5819         $outfd->close();
5820     };
5821     my $err = $@;
5822
5823     if ($err) {
5824
5825         unlink $tmpfn;
5826
5827         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5828
5829         die $err;
5830     }
5831
5832     rmtree $tmpdir;
5833
5834     rename $tmpfn, $conffile ||
5835         die "unable to commit configuration file '$conffile'\n";
5836
5837     PVE::Cluster::cfs_update(); # make sure we read new file
5838
5839     eval { rescan($vmid, 1); };
5840     warn $@ if $@;
5841 };
5842
5843 sub foreach_storage_used_by_vm {
5844     my ($conf, $func) = @_;
5845
5846     my $sidhash = {};
5847
5848     foreach_drive($conf, sub {
5849         my ($ds, $drive) = @_;
5850         return if drive_is_cdrom($drive);
5851
5852         my $volid = $drive->{file};
5853
5854         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5855         $sidhash->{$sid} = $sid if $sid;
5856     });
5857
5858     foreach my $sid (sort keys %$sidhash) {
5859         &$func($sid);
5860     }
5861 }
5862
5863 sub do_snapshots_with_qemu {
5864     my ($storecfg, $volid) = @_;
5865
5866     my $storage_name = PVE::Storage::parse_volume_id($volid);
5867
5868     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5869         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5870         return 1;
5871     }
5872
5873     if ($volid =~ m/\.(qcow2|qed)$/){
5874         return 1;
5875     }
5876
5877     return undef;
5878 }
5879
5880 sub qga_check_running {
5881     my ($vmid) = @_;
5882
5883     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5884     if ($@) {
5885         warn "Qemu Guest Agent are not running - $@";
5886         return 0;
5887     }
5888     return 1;
5889 }
5890
5891 sub template_create {
5892     my ($vmid, $conf, $disk) = @_;
5893
5894     my $storecfg = PVE::Storage::config();
5895
5896     foreach_drive($conf, sub {
5897         my ($ds, $drive) = @_;
5898
5899         return if drive_is_cdrom($drive);
5900         return if $disk && $ds ne $disk;
5901
5902         my $volid = $drive->{file};
5903         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5904
5905         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5906         $drive->{file} = $voliddst;
5907         $conf->{$ds} = print_drive($vmid, $drive);
5908         PVE::QemuConfig->write_config($vmid, $conf);
5909     });
5910 }
5911
5912 sub qemu_img_convert {
5913     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5914
5915     my $storecfg = PVE::Storage::config();
5916     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5917     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5918
5919     if ($src_storeid && $dst_storeid) {
5920
5921         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5922
5923         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5924         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5925
5926         my $src_format = qemu_img_format($src_scfg, $src_volname);
5927         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5928
5929         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5930         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5931
5932         my $cmd = [];
5933         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5934         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5935         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5936         if ($is_zero_initialized) {
5937             push @$cmd, "zeroinit:$dst_path";
5938         } else {
5939             push @$cmd, $dst_path;
5940         }
5941
5942         my $parser = sub {
5943             my $line = shift;
5944             if($line =~ m/\((\S+)\/100\%\)/){
5945                 my $percent = $1;
5946                 my $transferred = int($size * $percent / 100);
5947                 my $remaining = $size - $transferred;
5948
5949                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5950             }
5951
5952         };
5953
5954         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5955         my $err = $@;
5956         die "copy failed: $err" if $err;
5957     }
5958 }
5959
5960 sub qemu_img_format {
5961     my ($scfg, $volname) = @_;
5962
5963     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5964         return $1;
5965     } else {
5966         return "raw";
5967     }
5968 }
5969
5970 sub qemu_drive_mirror {
5971     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
5972
5973     $jobs = {} if !$jobs;
5974
5975     my $qemu_target;
5976     my $format;
5977     $jobs->{"drive-$drive"} = {};
5978
5979     if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
5980         my $server = $1;
5981         my $port = $2;
5982         my $exportname = $3;
5983
5984         $format = "nbd";
5985         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
5986         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
5987         my $cmd = ['socat', '-T30', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=5"];
5988
5989         my $pid = fork();
5990         if (!defined($pid)) {
5991             die "forking socat tunnel failed\n";
5992         } elsif ($pid == 0) {
5993             exec(@$cmd);
5994             warn "exec failed: $!\n";
5995             POSIX::_exit(-1);
5996         }
5997         $jobs->{"drive-$drive"}->{pid} = $pid;
5998
5999         my $timeout = 0;
6000         while (!-S $unixsocket) {
6001             die "nbd connection helper timed out\n"
6002                 if $timeout++ > 5;
6003             sleep 1;
6004         }
6005     } else {
6006         my $storecfg = PVE::Storage::config();
6007         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6008
6009         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6010
6011         $format = qemu_img_format($dst_scfg, $dst_volname);
6012
6013         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6014
6015         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6016     }
6017
6018     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6019     $opts->{format} = $format if $format;
6020
6021     print "drive mirror is starting for drive-$drive\n";
6022
6023     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6024
6025     if (my $err = $@) {
6026         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6027         die "mirroring error: $err";
6028     }
6029
6030     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6031 }
6032
6033 sub qemu_drive_mirror_monitor {
6034     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6035
6036     eval {
6037         my $err_complete = 0;
6038
6039         while (1) {
6040             die "storage migration timed out\n" if $err_complete > 300;
6041
6042             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6043
6044             my $running_mirror_jobs = {};
6045             foreach my $stat (@$stats) {
6046                 next if $stat->{type} ne 'mirror';
6047                 $running_mirror_jobs->{$stat->{device}} = $stat;
6048             }
6049
6050             my $readycounter = 0;
6051
6052             foreach my $job (keys %$jobs) {
6053
6054                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6055                     print "$job : finished\n";
6056                     delete $jobs->{$job};
6057                     next;
6058                 }
6059
6060                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6061
6062                 my $busy = $running_mirror_jobs->{$job}->{busy};
6063                 my $ready = $running_mirror_jobs->{$job}->{ready};
6064                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6065                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6066                     my $remaining = $total - $transferred;
6067                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6068
6069                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6070                 }
6071
6072                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6073             }
6074
6075             last if scalar(keys %$jobs) == 0;
6076
6077             if ($readycounter == scalar(keys %$jobs)) {
6078                 print "all mirroring jobs are ready \n";
6079                 last if $skipcomplete; #do the complete later
6080
6081                 if ($vmiddst && $vmiddst != $vmid) {
6082                     if ($qga) {
6083                         print "freeze filesystem\n";
6084                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6085                     } else {
6086                         print "suspend vm\n";
6087                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6088                     }
6089
6090                     # if we clone a disk for a new target vm, we don't switch the disk
6091                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6092
6093                     if ($qga) {
6094                         print "unfreeze filesystem\n";
6095                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6096                     } else {
6097                         print "resume vm\n";
6098                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6099                     }
6100
6101                     last;
6102                 } else {
6103
6104                     foreach my $job (keys %$jobs) {
6105                         # try to switch the disk if source and destination are on the same guest
6106                         print "$job: Completing block job...\n";
6107
6108                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6109                         if ($@ =~ m/cannot be completed/) {
6110                             print "$job: Block job cannot be completed, try again.\n";
6111                             $err_complete++;
6112                         }else {
6113                             print "$job: Completed successfully.\n";
6114                             $jobs->{$job}->{complete} = 1;
6115                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6116                         }
6117                     }
6118                 }
6119             }
6120             sleep 1;
6121         }
6122     };
6123     my $err = $@;
6124
6125     if ($err) {
6126         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6127         die "mirroring error: $err";
6128     }
6129
6130 }
6131
6132 sub qemu_blockjobs_cancel {
6133     my ($vmid, $jobs) = @_;
6134
6135     foreach my $job (keys %$jobs) {
6136         print "$job: Cancelling block job\n";
6137         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6138         $jobs->{$job}->{cancel} = 1;
6139     }
6140
6141     while (1) {
6142         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6143
6144         my $running_jobs = {};
6145         foreach my $stat (@$stats) {
6146             $running_jobs->{$stat->{device}} = $stat;
6147         }
6148
6149         foreach my $job (keys %$jobs) {
6150
6151             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6152                 print "$job: Done.\n";
6153                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6154                 delete $jobs->{$job};
6155             }
6156         }
6157
6158         last if scalar(keys %$jobs) == 0;
6159
6160         sleep 1;
6161     }
6162 }
6163
6164 sub qemu_blockjobs_finish_tunnel {
6165    my ($vmid, $job, $cpid) = @_;
6166
6167    return if !$cpid;
6168
6169    for (my $i = 1; $i < 20; $i++) {
6170         my $waitpid = waitpid($cpid, WNOHANG);
6171         last if (defined($waitpid) && ($waitpid == $cpid));
6172
6173         if ($i == 10) {
6174             kill(15, $cpid);
6175         } elsif ($i >= 15) {
6176             kill(9, $cpid);
6177         }
6178         sleep (1);
6179     }
6180     unlink "/run/qemu-server/$vmid.mirror-$job";
6181 }
6182
6183 sub clone_disk {
6184     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6185         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6186
6187     my $newvolid;
6188
6189     if (!$full) {
6190         print "create linked clone of drive $drivename ($drive->{file})\n";
6191         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6192         push @$newvollist, $newvolid;
6193     } else {
6194
6195         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6196         $storeid = $storage if $storage;
6197
6198         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6199         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6200
6201         print "create full clone of drive $drivename ($drive->{file})\n";
6202         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, undef, ($size/1024));
6203         push @$newvollist, $newvolid;
6204
6205         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6206
6207         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6208         if (!$running || $snapname) {
6209             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6210         } else {
6211
6212             my $kvmver = get_running_qemu_version ($vmid);
6213             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6214                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6215                     if $drive->{iothread};
6216             }
6217
6218             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6219         }
6220     }
6221
6222     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6223
6224     my $disk = $drive;
6225     $disk->{format} = undef;
6226     $disk->{file} = $newvolid;
6227     $disk->{size} = $size;
6228
6229     return $disk;
6230 }
6231
6232 # this only works if VM is running
6233 sub get_current_qemu_machine {
6234     my ($vmid) = @_;
6235
6236     my $cmd = { execute => 'query-machines', arguments => {} };
6237     my $res = vm_qmp_command($vmid, $cmd);
6238
6239     my ($current, $default);
6240     foreach my $e (@$res) {
6241         $default = $e->{name} if $e->{'is-default'};
6242         $current = $e->{name} if $e->{'is-current'};
6243     }
6244
6245     # fallback to the default machine if current is not supported by qemu
6246     return $current || $default || 'pc';
6247 }
6248
6249 sub get_running_qemu_version {
6250     my ($vmid) = @_;
6251     my $cmd = { execute => 'query-version', arguments => {} };
6252     my $res = vm_qmp_command($vmid, $cmd);
6253     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6254 }
6255
6256 sub qemu_machine_feature_enabled {
6257     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6258
6259     my $current_major;
6260     my $current_minor;
6261
6262     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6263
6264         $current_major = $3;
6265         $current_minor = $4;
6266
6267     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6268
6269         $current_major = $1;
6270         $current_minor = $2;
6271     }
6272
6273     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6274
6275
6276 }
6277
6278 sub qemu_machine_pxe {
6279     my ($vmid, $conf, $machine) = @_;
6280
6281     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6282
6283     foreach my $opt (keys %$conf) {
6284         next if $opt !~ m/^net(\d+)$/;
6285         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6286         next if !$net;
6287         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6288         return $machine.".pxe" if $romfile =~ m/pxe/;
6289         last;
6290     }
6291
6292     return $machine;
6293 }
6294
6295 sub qemu_use_old_bios_files {
6296     my ($machine_type) = @_;
6297
6298     return if !$machine_type;
6299
6300     my $use_old_bios_files = undef;
6301
6302     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6303         $machine_type = $1;
6304         $use_old_bios_files = 1;
6305     } else {
6306         my $kvmver = kvm_user_version();
6307         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6308         # load new efi bios files on migration. So this hack is required to allow
6309         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6310         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6311         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6312     }
6313
6314     return ($use_old_bios_files, $machine_type);
6315 }
6316
6317 sub create_efidisk {
6318     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6319
6320     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6321
6322     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6323     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6324     PVE::Storage::activate_volumes($storecfg, [$volid]);
6325
6326     my $path = PVE::Storage::path($storecfg, $volid);
6327     eval {
6328         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6329     };
6330     die "Copying EFI vars image failed: $@" if $@;
6331
6332     return ($volid, $vars_size);
6333 }
6334
6335 sub lspci {
6336
6337     my $devices = {};
6338
6339     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6340             my (undef, $id, $function) = @_;
6341             my $res = { id => $id, function => $function};
6342             push @{$devices->{$id}}, $res;
6343     });
6344
6345     # Entries should be sorted by functions.
6346     foreach my $id (keys %$devices) {
6347         my $dev = $devices->{$id};
6348         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6349     }
6350
6351     return $devices;
6352 }
6353
6354 sub vm_iothreads_list {
6355     my ($vmid) = @_;
6356
6357     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6358
6359     my $iothreads = {};
6360     foreach my $iothread (@$res) {
6361         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6362     }
6363
6364     return $iothreads;
6365 }
6366
6367 sub scsihw_infos {
6368     my ($conf, $drive) = @_;
6369
6370     my $maxdev = 0;
6371
6372     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6373         $maxdev = 7;
6374     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6375         $maxdev = 1;
6376     } else {
6377         $maxdev = 256;
6378     }
6379
6380     my $controller = int($drive->{index} / $maxdev);
6381     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6382
6383     return ($maxdev, $controller, $controller_prefix);
6384 }
6385
6386 sub add_hyperv_enlightenments {
6387     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6388
6389     return if $winversion < 6;
6390     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6391
6392     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6393
6394     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6395         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6396         push @$cpuFlags , 'hv_vapic';
6397         push @$cpuFlags , 'hv_time';
6398     } else {
6399         push @$cpuFlags , 'hv_spinlocks=0xffff';
6400     }
6401
6402     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6403         push @$cpuFlags , 'hv_reset';
6404         push @$cpuFlags , 'hv_vpindex';
6405         push @$cpuFlags , 'hv_runtime';
6406     }
6407
6408     if ($winversion >= 7) {
6409         push @$cpuFlags , 'hv_relaxed';
6410     }
6411 }
6412
6413 sub windows_version {
6414     my ($ostype) = @_;
6415
6416     return 0 if !$ostype;
6417
6418     my $winversion = 0;
6419
6420     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6421         $winversion = 5;
6422     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6423         $winversion = 6;
6424     } elsif ($ostype =~ m/^win(\d+)$/) {
6425         $winversion = $1;
6426     }
6427
6428     return $winversion;
6429 }
6430
6431 sub resolve_dst_disk_format {
6432         my ($storecfg, $storeid, $src_volname, $format) = @_;
6433         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6434
6435         if (!$format) {
6436             # if no target format is specified, use the source disk format as hint
6437             if ($src_volname) {
6438                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6439                 $format = qemu_img_format($scfg, $src_volname);
6440             } else {
6441                 return $defFormat;
6442             }
6443         }
6444
6445         # test if requested format is supported - else use default
6446         my $supported = grep { $_ eq $format } @$validFormats;
6447         $format = $defFormat if !$supported;
6448         return $format;
6449 }
6450
6451 sub resolve_first_disk {
6452     my $conf = shift;
6453     my @disks = PVE::QemuServer::valid_drive_names();
6454     my $firstdisk;
6455     foreach my $ds (reverse @disks) {
6456         next if !$conf->{$ds};
6457         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6458         next if PVE::QemuServer::drive_is_cdrom($disk);
6459         $firstdisk = $ds;
6460     }
6461     return $firstdisk;
6462 }
6463
6464 sub generate_smbios1_uuid {
6465     my ($uuid, $uuid_str);
6466     UUID::generate($uuid);
6467     UUID::unparse($uuid, $uuid_str);
6468     return "uuid=$uuid_str";
6469 }
6470
6471 # bash completion helper
6472
6473 sub complete_backup_archives {
6474     my ($cmdname, $pname, $cvalue) = @_;
6475
6476     my $cfg = PVE::Storage::config();
6477
6478     my $storeid;
6479
6480     if ($cvalue =~ m/^([^:]+):/) {
6481         $storeid = $1;
6482     }
6483
6484     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6485
6486     my $res = [];
6487     foreach my $id (keys %$data) {
6488         foreach my $item (@{$data->{$id}}) {
6489             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6490             push @$res, $item->{volid} if defined($item->{volid});
6491         }
6492     }
6493
6494     return $res;
6495 }
6496
6497 my $complete_vmid_full = sub {
6498     my ($running) = @_;
6499
6500     my $idlist = vmstatus();
6501
6502     my $res = [];
6503
6504     foreach my $id (keys %$idlist) {
6505         my $d = $idlist->{$id};
6506         if (defined($running)) {
6507             next if $d->{template};
6508             next if $running && $d->{status} ne 'running';
6509             next if !$running && $d->{status} eq 'running';
6510         }
6511         push @$res, $id;
6512
6513     }
6514     return $res;
6515 };
6516
6517 sub complete_vmid {
6518     return &$complete_vmid_full();
6519 }
6520
6521 sub complete_vmid_stopped {
6522     return &$complete_vmid_full(0);
6523 }
6524
6525 sub complete_vmid_running {
6526     return &$complete_vmid_full(1);
6527 }
6528
6529 sub complete_storage {
6530
6531     my $cfg = PVE::Storage::config();
6532     my $ids = $cfg->{ids};
6533
6534     my $res = [];
6535     foreach my $sid (keys %$ids) {
6536         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6537         next if !$ids->{$sid}->{content}->{images};
6538         push @$res, $sid;
6539     }
6540
6541     return $res;
6542 }
6543
6544 sub nbd_stop {
6545     my ($vmid) = @_;
6546
6547     vm_mon_cmd($vmid, 'nbd-server-stop');
6548 }
6549
6550 1;