PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42
  43 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  44
  45 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  46
  47 # Note about locking: we use flock on the config file protect
  48 # against concurent actions.
  49 # Aditionaly, we have a 'lock' setting in the config file. This
  50 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  51 # allowed when such lock is set. But you can ignore this kind of
  52 # lock with the --skiplock flag.
  53
  54 cfs_register_file('/qemu-server/',
  55                   \&parse_vm_config,
  56                   \&write_vm_config);
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  72     type => 'string',
  73     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  74     description => "The drive's backing file's data format.",
  75     optional => 1,
  76 });
  77
  78 #no warnings 'redefine';
  79
  80 sub cgroups_write {
  81    my ($controller, $vmid, $option, $value) = @_;
  82
  83    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  84    PVE::ProcFSTools::write_proc_entry($path, $value);
  85
  86 }
  87
  88 my $nodename = PVE::INotify::nodename();
  89
  90 mkdir "/etc/pve/nodes/$nodename";
  91 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  92 mkdir $confdir;
  93
  94 my $var_run_tmpdir = "/var/run/qemu-server";
  95 mkdir $var_run_tmpdir;
  96
  97 my $lock_dir = "/var/lock/qemu-server";
  98 mkdir $lock_dir;
  99
 100 my $pcisysfs = "/sys/bus/pci";
 101
 102 my $cpu_vendor_list = {
 103     # Intel CPUs
 104     486 => 'GenuineIntel',
 105     pentium => 'GenuineIntel',
 106     pentium2  => 'GenuineIntel',
 107     pentium3  => 'GenuineIntel',
 108     coreduo => 'GenuineIntel',
 109     core2duo => 'GenuineIntel',
 110     Conroe  => 'GenuineIntel',
 111     Penryn  => 'GenuineIntel',
 112     Nehalem  => 'GenuineIntel',
 113     'Nehalem-IBRS'  => 'GenuineIntel',
 114     Westmere => 'GenuineIntel',
 115     'Westmere-IBRS' => 'GenuineIntel',
 116     SandyBridge => 'GenuineIntel',
 117     'SandyBridge-IBRS' => 'GenuineIntel',
 118     IvyBridge => 'GenuineIntel',
 119     'IvyBridge-IBRS' => 'GenuineIntel',
 120     Haswell => 'GenuineIntel',
 121     'Haswell-IBRS' => 'GenuineIntel',
 122     'Haswell-noTSX' => 'GenuineIntel',
 123     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 124     Broadwell => 'GenuineIntel',
 125     'Broadwell-IBRS' => 'GenuineIntel',
 126     'Broadwell-noTSX' => 'GenuineIntel',
 127     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 128     'Skylake-Client' => 'GenuineIntel',
 129     'Skylake-Client-IBRS' => 'GenuineIntel',
 130     'Skylake-Server' => 'GenuineIntel',
 131     'Skylake-Server-IBRS' => 'GenuineIntel',
 132
 133     # AMD CPUs
 134     athlon => 'AuthenticAMD',
 135     phenom  => 'AuthenticAMD',
 136     Opteron_G1  => 'AuthenticAMD',
 137     Opteron_G2  => 'AuthenticAMD',
 138     Opteron_G3  => 'AuthenticAMD',
 139     Opteron_G4  => 'AuthenticAMD',
 140     Opteron_G5  => 'AuthenticAMD',
 141
 142     # generic types, use vendor from host node
 143     host => 'default',
 144     kvm32 => 'default',
 145     kvm64 => 'default',
 146     qemu32 => 'default',
 147     qemu64 => 'default',
 148     max => 'default',
 149 };
 150
 151 my $cpu_fmt = {
 152     cputype => {
 153         description => "Emulated CPU type.",
 154         type => 'string',
 155         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 156         default => 'kvm64',
 157         default_key => 1,
 158     },
 159     hidden => {
 160         description => "Do not identify as a KVM virtual machine.",
 161         type => 'boolean',
 162         optional => 1,
 163         default => 0
 164     },
 165     flags => {
 166         description => "Override CPU flags. Currently only the 'pcid' flag is supported."
 167                      . " Use '+pcid' or '-pcid' to enable or disable."
 168                      . " This takes precedence over flags coming from the cpu type or changed implicitly via the OS type.",
 169         format_description => 'flaglist',
 170         type => 'string',
 171         pattern => '[+-]pcid',
 172         optional => 1,
 173         default => '',
 174     },
 175 };
 176
 177 my $watchdog_fmt = {
 178     model => {
 179         default_key => 1,
 180         type => 'string',
 181         enum => [qw(i6300esb ib700)],
 182         description => "Watchdog type to emulate.",
 183         default => 'i6300esb',
 184         optional => 1,
 185     },
 186     action => {
 187         type => 'string',
 188         enum => [qw(reset shutdown poweroff pause debug none)],
 189         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 190         optional => 1,
 191     },
 192 };
 193 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 194
 195 my $confdesc = {
 196     onboot => {
 197         optional => 1,
 198         type => 'boolean',
 199         description => "Specifies whether a VM will be started during system bootup.",
 200         default => 0,
 201     },
 202     autostart => {
 203         optional => 1,
 204         type => 'boolean',
 205         description => "Automatic restart after crash (currently ignored).",
 206         default => 0,
 207     },
 208     hotplug => {
 209         optional => 1,
 210         type => 'string', format => 'pve-hotplug-features',
 211         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 212         default => 'network,disk,usb',
 213     },
 214     reboot => {
 215         optional => 1,
 216         type => 'boolean',
 217         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 218         default => 1,
 219     },
 220     lock => {
 221         optional => 1,
 222         type => 'string',
 223         description => "Lock/unlock the VM.",
 224         enum => [qw(migrate backup snapshot rollback)],
 225     },
 226     cpulimit => {
 227         optional => 1,
 228         type => 'number',
 229         description => "Limit of CPU usage.",
 230         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 231         minimum => 0,
 232         maximum => 128,
 233         default => 0,
 234     },
 235     cpuunits => {
 236         optional => 1,
 237         type => 'integer',
 238         description => "CPU weight for a VM.",
 239         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 240         minimum => 2,
 241         maximum => 262144,
 242         default => 1024,
 243     },
 244     memory => {
 245         optional => 1,
 246         type => 'integer',
 247         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 248         minimum => 16,
 249         default => 512,
 250     },
 251     balloon => {
 252         optional => 1,
 253         type => 'integer',
 254         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 255         minimum => 0,
 256     },
 257     shares => {
 258         optional => 1,
 259         type => 'integer',
 260         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 261         minimum => 0,
 262         maximum => 50000,
 263         default => 1000,
 264     },
 265     keyboard => {
 266         optional => 1,
 267         type => 'string',
 268         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.".
 269                        "It should not be necessary to set it.",
 270         enum => PVE::Tools::kvmkeymaplist(),
 271         default => undef,
 272     },
 273     name => {
 274         optional => 1,
 275         type => 'string', format => 'dns-name',
 276         description => "Set a name for the VM. Only used on the configuration web interface.",
 277     },
 278     scsihw => {
 279         optional => 1,
 280         type => 'string',
 281         description => "SCSI controller model",
 282         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 283         default => 'lsi',
 284     },
 285     description => {
 286         optional => 1,
 287         type => 'string',
 288         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 289     },
 290     ostype => {
 291         optional => 1,
 292         type => 'string',
 293         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 294         description => "Specify guest operating system.",
 295         verbose_description => <<EODESC,
 296 Specify guest operating system. This is used to enable special
 297 optimization/features for specific operating systems:
 298
 299 [horizontal]
 300 other;; unspecified OS
 301 wxp;; Microsoft Windows XP
 302 w2k;; Microsoft Windows 2000
 303 w2k3;; Microsoft Windows 2003
 304 w2k8;; Microsoft Windows 2008
 305 wvista;; Microsoft Windows Vista
 306 win7;; Microsoft Windows 7
 307 win8;; Microsoft Windows 8/2012/2012r2
 308 win10;; Microsoft Windows 10/2016
 309 l24;; Linux 2.4 Kernel
 310 l26;; Linux 2.6/3.X Kernel
 311 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 312 EODESC
 313     },
 314     boot => {
 315         optional => 1,
 316         type => 'string',
 317         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 318         pattern => '[acdn]{1,4}',
 319         default => 'cdn',
 320     },
 321     bootdisk => {
 322         optional => 1,
 323         type => 'string', format => 'pve-qm-bootdisk',
 324         description => "Enable booting from specified disk.",
 325         pattern => '(ide|sata|scsi|virtio)\d+',
 326     },
 327     smp => {
 328         optional => 1,
 329         type => 'integer',
 330         description => "The number of CPUs. Please use option -sockets instead.",
 331         minimum => 1,
 332         default => 1,
 333     },
 334     sockets => {
 335         optional => 1,
 336         type => 'integer',
 337         description => "The number of CPU sockets.",
 338         minimum => 1,
 339         default => 1,
 340     },
 341     cores => {
 342         optional => 1,
 343         type => 'integer',
 344         description => "The number of cores per socket.",
 345         minimum => 1,
 346         default => 1,
 347     },
 348     numa => {
 349         optional => 1,
 350         type => 'boolean',
 351         description => "Enable/disable NUMA.",
 352         default => 0,
 353     },
 354     hugepages => {
 355         optional => 1,
 356         type => 'string',
 357         description => "Enable/disable hugepages memory.",
 358         enum => [qw(any 2 1024)],
 359     },
 360     vcpus => {
 361         optional => 1,
 362         type => 'integer',
 363         description => "Number of hotplugged vcpus.",
 364         minimum => 1,
 365         default => 0,
 366     },
 367     acpi => {
 368         optional => 1,
 369         type => 'boolean',
 370         description => "Enable/disable ACPI.",
 371         default => 1,
 372     },
 373     agent => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Enable/disable Qemu GuestAgent.",
 377         default => 0,
 378     },
 379     kvm => {
 380         optional => 1,
 381         type => 'boolean',
 382         description => "Enable/disable KVM hardware virtualization.",
 383         default => 1,
 384     },
 385     tdf => {
 386         optional => 1,
 387         type => 'boolean',
 388         description => "Enable/disable time drift fix.",
 389         default => 0,
 390     },
 391     localtime => {
 392         optional => 1,
 393         type => 'boolean',
 394         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 395     },
 396     freeze => {
 397         optional => 1,
 398         type => 'boolean',
 399         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 400     },
 401     vga => {
 402         optional => 1,
 403         type => 'string',
 404         description => "Select the VGA type.",
 405         verbose_description => "Select the VGA type. If you want to use high resolution" .
 406             " modes (>= 1280x1024x16) then you should use the options " .
 407             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 408             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 409             "display sever. For win* OS you can select how many independent " .
 410             "displays you want, Linux guests can add displays them self. " .
 411             "You can also run without any graphic card, using a serial device" .
 412             " as terminal.",
 413         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 414     },
 415     watchdog => {
 416         optional => 1,
 417         type => 'string', format => 'pve-qm-watchdog',
 418         description => "Create a virtual hardware watchdog device.",
 419         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 420             " (by a guest action), the watchdog must be periodically polled " .
 421             "by an agent inside the guest or else the watchdog will reset " .
 422             "the guest (or execute the respective action specified)",
 423     },
 424     startdate => {
 425         optional => 1,
 426         type => 'string',
 427         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 428         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 429         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 430         default => 'now',
 431     },
 432     startup =>  get_standard_option('pve-startup-order'),
 433     template => {
 434         optional => 1,
 435         type => 'boolean',
 436         description => "Enable/disable Template.",
 437         default => 0,
 438     },
 439     args => {
 440         optional => 1,
 441         type => 'string',
 442         description => "Arbitrary arguments passed to kvm.",
 443         verbose_description => <<EODESCR,
 444 Arbitrary arguments passed to kvm, for example:
 445
 446 args: -no-reboot -no-hpet
 447
 448 NOTE: this option is for experts only.
 449 EODESCR
 450     },
 451     tablet => {
 452         optional => 1,
 453         type => 'boolean',
 454         default => 1,
 455         description => "Enable/disable the USB tablet device.",
 456         verbose_description => "Enable/disable the USB tablet device. This device is " .
 457             "usually needed to allow absolute mouse positioning with VNC. " .
 458             "Else the mouse runs out of sync with normal VNC clients. " .
 459             "If you're running lots of console-only guests on one host, " .
 460             "you may consider disabling this to save some context switches. " .
 461             "This is turned off by default if you use spice (-vga=qxl).",
 462     },
 463     migrate_speed => {
 464         optional => 1,
 465         type => 'integer',
 466         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 467         minimum => 0,
 468         default => 0,
 469     },
 470     migrate_downtime => {
 471         optional => 1,
 472         type => 'number',
 473         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 474         minimum => 0,
 475         default => 0.1,
 476     },
 477     cdrom => {
 478         optional => 1,
 479         type => 'string', format => 'pve-qm-ide',
 480         typetext => '<volume>',
 481         description => "This is an alias for option -ide2",
 482     },
 483     cpu => {
 484         optional => 1,
 485         description => "Emulated CPU type.",
 486         type => 'string',
 487         format => $cpu_fmt,
 488     },
 489     parent => get_standard_option('pve-snapshot-name', {
 490         optional => 1,
 491         description => "Parent snapshot name. This is used internally, and should not be modified.",
 492     }),
 493     snaptime => {
 494         optional => 1,
 495         description => "Timestamp for snapshots.",
 496         type => 'integer',
 497         minimum => 0,
 498     },
 499     vmstate => {
 500         optional => 1,
 501         type => 'string', format => 'pve-volume-id',
 502         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 503     },
 504     vmstatestorage => get_standard_option('pve-storage-id', {
 505         description => "Default storage for VM state volumes/files.",
 506         optional => 1,
 507     }),
 508     machine => {
 509         description => "Specific the Qemu machine type.",
 510         type => 'string',
 511         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 512         maxLength => 40,
 513         optional => 1,
 514     },
 515     smbios1 => {
 516         description => "Specify SMBIOS type 1 fields.",
 517         type => 'string', format => 'pve-qm-smbios1',
 518         maxLength => 256,
 519         optional => 1,
 520     },
 521     protection => {
 522         optional => 1,
 523         type => 'boolean',
 524         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 525         default => 0,
 526     },
 527     bios => {
 528         optional => 1,
 529         type => 'string',
 530         enum => [ qw(seabios ovmf) ],
 531         description => "Select BIOS implementation.",
 532         default => 'seabios',
 533     },
 534 };
 535
 536 # what about other qemu settings ?
 537 #cpu => 'string',
 538 #machine => 'string',
 539 #fda => 'file',
 540 #fdb => 'file',
 541 #mtdblock => 'file',
 542 #sd => 'file',
 543 #pflash => 'file',
 544 #snapshot => 'bool',
 545 #bootp => 'file',
 546 ##tftp => 'dir',
 547 ##smb => 'dir',
 548 #kernel => 'file',
 549 #append => 'string',
 550 #initrd => 'file',
 551 ##soundhw => 'string',
 552
 553 while (my ($k, $v) = each %$confdesc) {
 554     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 555 }
 556
 557 my $MAX_IDE_DISKS = 4;
 558 my $MAX_SCSI_DISKS = 14;
 559 my $MAX_VIRTIO_DISKS = 16;
 560 my $MAX_SATA_DISKS = 6;
 561 my $MAX_USB_DEVICES = 5;
 562 my $MAX_NETS = 32;
 563 my $MAX_UNUSED_DISKS = 8;
 564 my $MAX_HOSTPCI_DEVICES = 4;
 565 my $MAX_SERIAL_PORTS = 4;
 566 my $MAX_PARALLEL_PORTS = 3;
 567 my $MAX_NUMA = 8;
 568
 569 my $numa_fmt = {
 570     cpus => {
 571         type => "string",
 572         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 573         description => "CPUs accessing this NUMA node.",
 574         format_description => "id[-id];...",
 575     },
 576     memory => {
 577         type => "number",
 578         description => "Amount of memory this NUMA node provides.",
 579         optional => 1,
 580     },
 581     hostnodes => {
 582         type => "string",
 583         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 584         description => "Host NUMA nodes to use.",
 585         format_description => "id[-id];...",
 586         optional => 1,
 587     },
 588     policy => {
 589         type => 'string',
 590         enum => [qw(preferred bind interleave)],
 591         description => "NUMA allocation policy.",
 592         optional => 1,
 593     },
 594 };
 595 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 596 my $numadesc = {
 597     optional => 1,
 598     type => 'string', format => $numa_fmt,
 599     description => "NUMA topology.",
 600 };
 601 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 602
 603 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 604     $confdesc->{"numa$i"} = $numadesc;
 605 }
 606
 607 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 608                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 609                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 610 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 611
 612 my $net_fmt_bridge_descr = <<__EOD__;
 613 Bridge to attach the network device to. The Proxmox VE standard bridge
 614 is called 'vmbr0'.
 615
 616 If you do not specify a bridge, we create a kvm user (NATed) network
 617 device, which provides DHCP and DNS services. The following addresses
 618 are used:
 619
 620  10.0.2.2   Gateway
 621  10.0.2.3   DNS Server
 622  10.0.2.4   SMB Server
 623
 624 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 625 __EOD__
 626
 627 my $net_fmt = {
 628     macaddr => {
 629         type => 'string',
 630         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 631         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 632         format_description => "XX:XX:XX:XX:XX:XX",
 633         optional => 1,
 634     },
 635     model => {
 636         type => 'string',
 637         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 638         enum => $nic_model_list,
 639         default_key => 1,
 640     },
 641     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 642     bridge => {
 643         type => 'string',
 644         description => $net_fmt_bridge_descr,
 645         format_description => 'bridge',
 646         optional => 1,
 647     },
 648     queues => {
 649         type => 'integer',
 650         minimum => 0, maximum => 16,
 651         description => 'Number of packet queues to be used on the device.',
 652         optional => 1,
 653     },
 654     rate => {
 655         type => 'number',
 656         minimum => 0,
 657         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 658         optional => 1,
 659     },
 660     tag => {
 661         type => 'integer',
 662         minimum => 1, maximum => 4094,
 663         description => 'VLAN tag to apply to packets on this interface.',
 664         optional => 1,
 665     },
 666     trunks => {
 667         type => 'string',
 668         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 669         description => 'VLAN trunks to pass through this interface.',
 670         format_description => 'vlanid[;vlanid...]',
 671         optional => 1,
 672     },
 673     firewall => {
 674         type => 'boolean',
 675         description => 'Whether this interface should be protected by the firewall.',
 676         optional => 1,
 677     },
 678     link_down => {
 679         type => 'boolean',
 680         description => 'Whether this interface should be disconnected (like pulling the plug).',
 681         optional => 1,
 682     },
 683 };
 684
 685 my $netdesc = {
 686     optional => 1,
 687     type => 'string', format => $net_fmt,
 688     description => "Specify network devices.",
 689 };
 690
 691 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 692
 693 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 694     $confdesc->{"net$i"} = $netdesc;
 695 }
 696
 697 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 698 sub verify_volume_id_or_qm_path {
 699     my ($volid, $noerr) = @_;
 700
 701     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 702         return $volid;
 703     }
 704
 705     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 706     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 707     if ($@) {
 708         return undef if $noerr;
 709         die $@;
 710     }
 711     return $volid;
 712 }
 713
 714 my $drivename_hash;
 715
 716 my %drivedesc_base = (
 717     volume => { alias => 'file' },
 718     file => {
 719         type => 'string',
 720         format => 'pve-volume-id-or-qm-path',
 721         default_key => 1,
 722         format_description => 'volume',
 723         description => "The drive's backing volume.",
 724     },
 725     media => {
 726         type => 'string',
 727         enum => [qw(cdrom disk)],
 728         description => "The drive's media type.",
 729         default => 'disk',
 730         optional => 1
 731     },
 732     cyls => {
 733         type => 'integer',
 734         description => "Force the drive's physical geometry to have a specific cylinder count.",
 735         optional => 1
 736     },
 737     heads => {
 738         type => 'integer',
 739         description => "Force the drive's physical geometry to have a specific head count.",
 740         optional => 1
 741     },
 742     secs => {
 743         type => 'integer',
 744         description => "Force the drive's physical geometry to have a specific sector count.",
 745         optional => 1
 746     },
 747     trans => {
 748         type => 'string',
 749         enum => [qw(none lba auto)],
 750         description => "Force disk geometry bios translation mode.",
 751         optional => 1,
 752     },
 753     snapshot => {
 754         type => 'boolean',
 755         description => "Whether the drive should be included when making snapshots.",
 756         optional => 1,
 757     },
 758     cache => {
 759         type => 'string',
 760         enum => [qw(none writethrough writeback unsafe directsync)],
 761         description => "The drive's cache mode",
 762         optional => 1,
 763     },
 764     format => get_standard_option('pve-qm-image-format'),
 765     size => {
 766         type => 'string',
 767         format => 'disk-size',
 768         format_description => 'DiskSize',
 769         description => "Disk size. This is purely informational and has no effect.",
 770         optional => 1,
 771     },
 772     backup => {
 773         type => 'boolean',
 774         description => "Whether the drive should be included when making backups.",
 775         optional => 1,
 776     },
 777     replicate => {
 778         type => 'boolean',
 779         description => 'Whether the drive should considered for replication jobs.',
 780         optional => 1,
 781         default => 1,
 782     },
 783     rerror => {
 784         type => 'string',
 785         enum => [qw(ignore report stop)],
 786         description => 'Read error action.',
 787         optional => 1,
 788     },
 789     werror => {
 790         type => 'string',
 791         enum => [qw(enospc ignore report stop)],
 792         description => 'Write error action.',
 793         optional => 1,
 794     },
 795     aio => {
 796         type => 'string',
 797         enum => [qw(native threads)],
 798         description => 'AIO type to use.',
 799         optional => 1,
 800     },
 801     discard => {
 802         type => 'string',
 803         enum => [qw(ignore on)],
 804         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 805         optional => 1,
 806     },
 807     detect_zeroes => {
 808         type => 'boolean',
 809         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 810         optional => 1,
 811     },
 812     serial => {
 813         type => 'string',
 814         format => 'urlencoded',
 815         format_description => 'serial',
 816         maxLength => 20*3, # *3 since it's %xx url enoded
 817         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 818         optional => 1,
 819     }
 820 );
 821
 822 my %iothread_fmt = ( iothread => {
 823         type => 'boolean',
 824         description => "Whether to use iothreads for this drive",
 825         optional => 1,
 826 });
 827
 828 my %model_fmt = (
 829     model => {
 830         type => 'string',
 831         format => 'urlencoded',
 832         format_description => 'model',
 833         maxLength => 40*3, # *3 since it's %xx url enoded
 834         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 835         optional => 1,
 836     },
 837 );
 838
 839 my %queues_fmt = (
 840     queues => {
 841         type => 'integer',
 842         description => "Number of queues.",
 843         minimum => 2,
 844         optional => 1
 845     }
 846 );
 847
 848 my %scsiblock_fmt = (
 849     scsiblock => {
 850         type => 'boolean',
 851         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 852         optional => 1,
 853         default => 0,
 854     },
 855 );
 856
 857 my $add_throttle_desc = sub {
 858     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 859     my $d = {
 860         type => $type,
 861         format_description => $unit,
 862         description => "Maximum $what in $longunit.",
 863         optional => 1,
 864     };
 865     $d->{minimum} = $minimum if defined($minimum);
 866     $drivedesc_base{$key} = $d;
 867 };
 868 # throughput: (leaky bucket)
 869 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 870 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 871 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 872 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 873 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 874 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 875 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 876 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 877 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 878
 879 # pools: (pool of IO before throttling starts taking effect)
 880 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 881 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 882 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 883 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 884 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 885 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 886
 887 # burst lengths
 888 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 889 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 890 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 891 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 892 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 893 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 894
 895 # legacy support
 896 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
 897 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
 898 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
 899 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
 900
 901 my $ide_fmt = {
 902     %drivedesc_base,
 903     %model_fmt,
 904 };
 905 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 906
 907 my $idedesc = {
 908     optional => 1,
 909     type => 'string', format => $ide_fmt,
 910     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 911 };
 912 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 913
 914 my $scsi_fmt = {
 915     %drivedesc_base,
 916     %iothread_fmt,
 917     %queues_fmt,
 918     %scsiblock_fmt,
 919 };
 920 my $scsidesc = {
 921     optional => 1,
 922     type => 'string', format => $scsi_fmt,
 923     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 924 };
 925 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 926
 927 my $sata_fmt = {
 928     %drivedesc_base,
 929 };
 930 my $satadesc = {
 931     optional => 1,
 932     type => 'string', format => $sata_fmt,
 933     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 934 };
 935 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 936
 937 my $virtio_fmt = {
 938     %drivedesc_base,
 939     %iothread_fmt,
 940 };
 941 my $virtiodesc = {
 942     optional => 1,
 943     type => 'string', format => $virtio_fmt,
 944     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 945 };
 946 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 947
 948 my $alldrive_fmt = {
 949     %drivedesc_base,
 950     %iothread_fmt,
 951     %model_fmt,
 952     %queues_fmt,
 953     %scsiblock_fmt,
 954 };
 955
 956 my $efidisk_fmt = {
 957     volume => { alias => 'file' },
 958     file => {
 959         type => 'string',
 960         format => 'pve-volume-id-or-qm-path',
 961         default_key => 1,
 962         format_description => 'volume',
 963         description => "The drive's backing volume.",
 964     },
 965     format => get_standard_option('pve-qm-image-format'),
 966     size => {
 967         type => 'string',
 968         format => 'disk-size',
 969         format_description => 'DiskSize',
 970         description => "Disk size. This is purely informational and has no effect.",
 971         optional => 1,
 972     },
 973 };
 974
 975 my $efidisk_desc = {
 976     optional => 1,
 977     type => 'string', format => $efidisk_fmt,
 978     description => "Configure a Disk for storing EFI vars",
 979 };
 980
 981 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 982
 983 my $usb_fmt = {
 984     host => {
 985         default_key => 1,
 986         type => 'string', format => 'pve-qm-usb-device',
 987         format_description => 'HOSTUSBDEVICE|spice',
 988         description => <<EODESCR,
 989 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 990
 991  'bus-port(.port)*' (decimal numbers) or
 992  'vendor_id:product_id' (hexadeciaml numbers) or
 993  'spice'
 994
 995 You can use the 'lsusb -t' command to list existing usb devices.
 996
 997 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 998
 999 The value 'spice' can be used to add a usb redirection devices for spice.
1000 EODESCR
1001     },
1002     usb3 => {
1003         optional => 1,
1004         type => 'boolean',
1005         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1006         default => 0,
1007     },
1008 };
1009
1010 my $usbdesc = {
1011     optional => 1,
1012     type => 'string', format => $usb_fmt,
1013     description => "Configure an USB device (n is 0 to 4).",
1014 };
1015 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1016
1017 # NOTE: the match-groups of this regex are used in parse_hostpci
1018 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1019 my $hostpci_fmt = {
1020     host => {
1021         default_key => 1,
1022         type => 'string',
1023         pattern => qr/$PCIRE(;$PCIRE)*/,
1024         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1025         description => <<EODESCR,
1026 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1027 of PCI virtual functions of the host. HOSTPCIID syntax is:
1028
1029 'bus:dev.func' (hexadecimal numbers)
1030
1031 You can us the 'lspci' command to list existing PCI devices.
1032 EODESCR
1033     },
1034     rombar => {
1035         type => 'boolean',
1036         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1037         optional => 1,
1038         default => 1,
1039     },
1040     romfile => {
1041         type => 'string',
1042         pattern => '[^,;]+',
1043         format_description => 'string',
1044         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1045         optional => 1,
1046     },
1047     pcie => {
1048         type => 'boolean',
1049         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1050         optional => 1,
1051         default => 0,
1052     },
1053     'x-vga' => {
1054         type => 'boolean',
1055         description =>  "Enable vfio-vga device support.",
1056         optional => 1,
1057         default => 0,
1058     },
1059 };
1060 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1061
1062 my $hostpcidesc = {
1063         optional => 1,
1064         type => 'string', format => 'pve-qm-hostpci',
1065         description => "Map host PCI devices into guest.",
1066         verbose_description =>  <<EODESCR,
1067 Map host PCI devices into guest.
1068
1069 NOTE: This option allows direct access to host hardware. So it is no longer
1070 possible to migrate such machines - use with special care.
1071
1072 CAUTION: Experimental! User reported problems with this option.
1073 EODESCR
1074 };
1075 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1076
1077 my $serialdesc = {
1078         optional => 1,
1079         type => 'string',
1080         pattern => '(/dev/.+|socket)',
1081         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1082         verbose_description =>  <<EODESCR,
1083 Create a serial device inside the VM (n is 0 to 3), and pass through a
1084 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1085 host side (use 'qm terminal' to open a terminal connection).
1086
1087 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1088
1089 CAUTION: Experimental! User reported problems with this option.
1090 EODESCR
1091 };
1092
1093 my $paralleldesc= {
1094         optional => 1,
1095         type => 'string',
1096         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1097         description =>  "Map host parallel devices (n is 0 to 2).",
1098         verbose_description =>  <<EODESCR,
1099 Map host parallel devices (n is 0 to 2).
1100
1101 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1102
1103 CAUTION: Experimental! User reported problems with this option.
1104 EODESCR
1105 };
1106
1107 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1108     $confdesc->{"parallel$i"} = $paralleldesc;
1109 }
1110
1111 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1112     $confdesc->{"serial$i"} = $serialdesc;
1113 }
1114
1115 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1116     $confdesc->{"hostpci$i"} = $hostpcidesc;
1117 }
1118
1119 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1120     $drivename_hash->{"ide$i"} = 1;
1121     $confdesc->{"ide$i"} = $idedesc;
1122 }
1123
1124 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1125     $drivename_hash->{"sata$i"} = 1;
1126     $confdesc->{"sata$i"} = $satadesc;
1127 }
1128
1129 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1130     $drivename_hash->{"scsi$i"} = 1;
1131     $confdesc->{"scsi$i"} = $scsidesc ;
1132 }
1133
1134 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1135     $drivename_hash->{"virtio$i"} = 1;
1136     $confdesc->{"virtio$i"} = $virtiodesc;
1137 }
1138
1139 $drivename_hash->{efidisk0} = 1;
1140 $confdesc->{efidisk0} = $efidisk_desc;
1141
1142 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1143     $confdesc->{"usb$i"} = $usbdesc;
1144 }
1145
1146 my $unuseddesc = {
1147     optional => 1,
1148     type => 'string', format => 'pve-volume-id',
1149     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1150 };
1151
1152 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1153     $confdesc->{"unused$i"} = $unuseddesc;
1154 }
1155
1156 my $kvm_api_version = 0;
1157
1158 sub kvm_version {
1159
1160     return $kvm_api_version if $kvm_api_version;
1161
1162     my $fh = IO::File->new("</dev/kvm") ||
1163         return 0;
1164
1165     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1166         $kvm_api_version = $v;
1167     }
1168
1169     $fh->close();
1170
1171     return  $kvm_api_version;
1172 }
1173
1174 my $kvm_user_version;
1175
1176 sub kvm_user_version {
1177
1178     return $kvm_user_version if $kvm_user_version;
1179
1180     $kvm_user_version = 'unknown';
1181
1182     my $code = sub {
1183         my $line = shift;
1184         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1185             $kvm_user_version = $2;
1186         }
1187     };
1188
1189     eval { run_command("kvm -version", outfunc => $code); };
1190     warn $@ if $@;
1191
1192     return $kvm_user_version;
1193
1194 }
1195
1196 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1197
1198 sub valid_drive_names {
1199     # order is important - used to autoselect boot disk
1200     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1201             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1202             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1203             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1204             'efidisk0');
1205 }
1206
1207 sub is_valid_drivename {
1208     my $dev = shift;
1209
1210     return defined($drivename_hash->{$dev});
1211 }
1212
1213 sub option_exists {
1214     my $key = shift;
1215     return defined($confdesc->{$key});
1216 }
1217
1218 sub nic_models {
1219     return $nic_model_list;
1220 }
1221
1222 sub os_list_description {
1223
1224     return {
1225         other => 'Other',
1226         wxp => 'Windows XP',
1227         w2k => 'Windows 2000',
1228         w2k3 =>, 'Windows 2003',
1229         w2k8 => 'Windows 2008',
1230         wvista => 'Windows Vista',
1231         win7 => 'Windows 7',
1232         win8 => 'Windows 8/2012',
1233         win10 => 'Windows 10/2016',
1234         l24 => 'Linux 2.4',
1235         l26 => 'Linux 2.6',
1236     };
1237 }
1238
1239 my $cdrom_path;
1240
1241 sub get_cdrom_path {
1242
1243     return  $cdrom_path if $cdrom_path;
1244
1245     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1246     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1247     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1248 }
1249
1250 sub get_iso_path {
1251     my ($storecfg, $vmid, $cdrom) = @_;
1252
1253     if ($cdrom eq 'cdrom') {
1254         return get_cdrom_path();
1255     } elsif ($cdrom eq 'none') {
1256         return '';
1257     } elsif ($cdrom =~ m|^/|) {
1258         return $cdrom;
1259     } else {
1260         return PVE::Storage::path($storecfg, $cdrom);
1261     }
1262 }
1263
1264 # try to convert old style file names to volume IDs
1265 sub filename_to_volume_id {
1266     my ($vmid, $file, $media) = @_;
1267
1268     if (!($file eq 'none' || $file eq 'cdrom' ||
1269           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1270
1271         return undef if $file =~ m|/|;
1272
1273         if ($media && $media eq 'cdrom') {
1274             $file = "local:iso/$file";
1275         } else {
1276             $file = "local:$vmid/$file";
1277         }
1278     }
1279
1280     return $file;
1281 }
1282
1283 sub verify_media_type {
1284     my ($opt, $vtype, $media) = @_;
1285
1286     return if !$media;
1287
1288     my $etype;
1289     if ($media eq 'disk') {
1290         $etype = 'images';
1291     } elsif ($media eq 'cdrom') {
1292         $etype = 'iso';
1293     } else {
1294         die "internal error";
1295     }
1296
1297     return if ($vtype eq $etype);
1298
1299     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1300 }
1301
1302 sub cleanup_drive_path {
1303     my ($opt, $storecfg, $drive) = @_;
1304
1305     # try to convert filesystem paths to volume IDs
1306
1307     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1308         ($drive->{file} !~ m|^/dev/.+|) &&
1309         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1310         ($drive->{file} !~ m/^\d+$/)) {
1311         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1312         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1313         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1314         verify_media_type($opt, $vtype, $drive->{media});
1315         $drive->{file} = $volid;
1316     }
1317
1318     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1319 }
1320
1321 sub parse_hotplug_features {
1322     my ($data) = @_;
1323
1324     my $res = {};
1325
1326     return $res if $data eq '0';
1327
1328     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1329
1330     foreach my $feature (PVE::Tools::split_list($data)) {
1331         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1332             $res->{$1} = 1;
1333         } else {
1334             die "invalid hotplug feature '$feature'\n";
1335         }
1336     }
1337     return $res;
1338 }
1339
1340 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1341 sub pve_verify_hotplug_features {
1342     my ($value, $noerr) = @_;
1343
1344     return $value if parse_hotplug_features($value);
1345
1346     return undef if $noerr;
1347
1348     die "unable to parse hotplug option\n";
1349 }
1350
1351 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1352 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1353 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1354 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1355 #        [,iothread=on][,serial=serial][,model=model]
1356
1357 sub parse_drive {
1358     my ($key, $data) = @_;
1359
1360     my ($interface, $index);
1361
1362     if ($key =~ m/^([^\d]+)(\d+)$/) {
1363         $interface = $1;
1364         $index = $2;
1365     } else {
1366         return undef;
1367     }
1368
1369     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1370                                      : $confdesc->{$key}->{format};
1371     if (!$desc) {
1372         warn "invalid drive key: $key\n";
1373         return undef;
1374     }
1375     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1376     return undef if !$res;
1377     $res->{interface} = $interface;
1378     $res->{index} = $index;
1379
1380     my $error = 0;
1381     foreach my $opt (qw(bps bps_rd bps_wr)) {
1382         if (my $bps = defined(delete $res->{$opt})) {
1383             if (defined($res->{"m$opt"})) {
1384                 warn "both $opt and m$opt specified\n";
1385                 ++$error;
1386                 next;
1387             }
1388             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1389         }
1390     }
1391
1392     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1393     for my $requirement (
1394         [mbps_max => 'mbps'],
1395         [mbps_rd_max => 'mbps_rd'],
1396         [mbps_wr_max => 'mbps_wr'],
1397         [miops_max => 'miops'],
1398         [miops_rd_max => 'miops_rd'],
1399         [miops_wr_max => 'miops_wr'],
1400         [bps_max_length => 'mbps_max'],
1401         [bps_rd_max_length => 'mbps_rd_max'],
1402         [bps_wr_max_length => 'mbps_wr_max'],
1403         [iops_max_length => 'iops_max'],
1404         [iops_rd_max_length => 'iops_rd_max'],
1405         [iops_wr_max_length => 'iops_wr_max']) {
1406         my ($option, $requires) = @$requirement;
1407         if ($res->{$option} && !$res->{$requires}) {
1408             warn "$option requires $requires\n";
1409             ++$error;
1410         }
1411     }
1412
1413     return undef if $error;
1414
1415     return undef if $res->{mbps_rd} && $res->{mbps};
1416     return undef if $res->{mbps_wr} && $res->{mbps};
1417     return undef if $res->{iops_rd} && $res->{iops};
1418     return undef if $res->{iops_wr} && $res->{iops};
1419
1420     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1421         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1422         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1423         return undef if $res->{interface} eq 'virtio';
1424     }
1425
1426     if (my $size = $res->{size}) {
1427         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1428     }
1429
1430     return $res;
1431 }
1432
1433 sub print_drive {
1434     my ($vmid, $drive) = @_;
1435     my $data = { %$drive };
1436     delete $data->{$_} for qw(index interface);
1437     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1438 }
1439
1440 sub scsi_inquiry {
1441     my($fh, $noerr) = @_;
1442
1443     my $SG_IO = 0x2285;
1444     my $SG_GET_VERSION_NUM = 0x2282;
1445
1446     my $versionbuf = "\x00" x 8;
1447     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1448     if (!$ret) {
1449         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1450         return undef;
1451     }
1452     my $version = unpack("I", $versionbuf);
1453     if ($version < 30000) {
1454         die "scsi generic interface too old\n"  if !$noerr;
1455         return undef;
1456     }
1457
1458     my $buf = "\x00" x 36;
1459     my $sensebuf = "\x00" x 8;
1460     my $cmd = pack("C x3 C x1", 0x12, 36);
1461
1462     # see /usr/include/scsi/sg.h
1463     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1464
1465     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1466                       length($sensebuf), 0, length($buf), $buf,
1467                       $cmd, $sensebuf, 6000);
1468
1469     $ret = ioctl($fh, $SG_IO, $packet);
1470     if (!$ret) {
1471         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1472         return undef;
1473     }
1474
1475     my @res = unpack($sg_io_hdr_t, $packet);
1476     if ($res[17] || $res[18]) {
1477         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1478         return undef;
1479     }
1480
1481     my $res = {};
1482     (my $byte0, my $byte1, $res->{vendor},
1483      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1484
1485     $res->{removable} = $byte1 & 128 ? 1 : 0;
1486     $res->{type} = $byte0 & 31;
1487
1488     return $res;
1489 }
1490
1491 sub path_is_scsi {
1492     my ($path) = @_;
1493
1494     my $fh = IO::File->new("+<$path") || return undef;
1495     my $res = scsi_inquiry($fh, 1);
1496     close($fh);
1497
1498     return $res;
1499 }
1500
1501 sub machine_type_is_q35 {
1502     my ($conf) = @_;
1503
1504     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1505 }
1506
1507 sub print_tabletdevice_full {
1508     my ($conf) = @_;
1509
1510     my $q35 = machine_type_is_q35($conf);
1511
1512     # we use uhci for old VMs because tablet driver was buggy in older qemu
1513     my $usbbus = $q35 ? "ehci" : "uhci";
1514
1515     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1516 }
1517
1518 sub print_drivedevice_full {
1519     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1520
1521     my $device = '';
1522     my $maxdev = 0;
1523
1524     if ($drive->{interface} eq 'virtio') {
1525         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1526         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1527         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1528     } elsif ($drive->{interface} eq 'scsi') {
1529
1530         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1531         my $unit = $drive->{index} % $maxdev;
1532         my $devicetype = 'hd';
1533         my $path = '';
1534         if (drive_is_cdrom($drive)) {
1535             $devicetype = 'cd';
1536         } else {
1537             if ($drive->{file} =~ m|^/|) {
1538                 $path = $drive->{file};
1539                 if (my $info = path_is_scsi($path)) {
1540                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1541                         $devicetype = 'block';
1542                     } elsif ($info->{type} == 1) { # tape
1543                         $devicetype = 'generic';
1544                     }
1545                 }
1546             } else {
1547                  $path = PVE::Storage::path($storecfg, $drive->{file});
1548             }
1549
1550             if($path =~ m/^iscsi\:\/\//){
1551                 $devicetype = 'generic';
1552             }
1553         }
1554
1555         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1556             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1557         } else {
1558             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1559         }
1560
1561     } elsif ($drive->{interface} eq 'ide'){
1562         $maxdev = 2;
1563         my $controller = int($drive->{index} / $maxdev);
1564         my $unit = $drive->{index} % $maxdev;
1565         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1566
1567         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1568         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1569             $model = URI::Escape::uri_unescape($model);
1570             $device .= ",model=$model";
1571         }
1572     } elsif ($drive->{interface} eq 'sata'){
1573         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1574         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1575         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1576     } elsif ($drive->{interface} eq 'usb') {
1577         die "implement me";
1578         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1579     } else {
1580         die "unsupported interface type";
1581     }
1582
1583     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1584
1585     return $device;
1586 }
1587
1588 sub get_initiator_name {
1589     my $initiator;
1590
1591     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1592     while (defined(my $line = <$fh>)) {
1593         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1594         $initiator = $1;
1595         last;
1596     }
1597     $fh->close();
1598
1599     return $initiator;
1600 }
1601
1602 sub print_drive_full {
1603     my ($storecfg, $vmid, $drive) = @_;
1604
1605     my $path;
1606     my $volid = $drive->{file};
1607     my $format;
1608
1609     if (drive_is_cdrom($drive)) {
1610         $path = get_iso_path($storecfg, $vmid, $volid);
1611     } else {
1612         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1613         if ($storeid) {
1614             $path = PVE::Storage::path($storecfg, $volid);
1615             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1616             $format = qemu_img_format($scfg, $volname);
1617         } else {
1618             $path = $volid;
1619             $format = "raw";
1620         }
1621    }
1622
1623     my $opts = '';
1624     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard);
1625     foreach my $o (@qemu_drive_options) {
1626         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1627     }
1628     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1629         my ($dir, $qmpname) = @$type;
1630         if (my $v = $drive->{"mbps$dir"}) {
1631             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1632         }
1633         if (my $v = $drive->{"mbps${dir}_max"}) {
1634             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1635         }
1636         if (my $v = $drive->{"bps${dir}_max_length"}) {
1637             $opts .= ",throttling.bps$qmpname-max-length=$v";
1638         }
1639         if (my $v = $drive->{"iops${dir}"}) {
1640             $opts .= ",throttling.iops$qmpname=$v";
1641         }
1642         if (my $v = $drive->{"iops${dir}_max"}) {
1643             $opts .= ",throttling.iops$qmpname-max=$v";
1644         }
1645         if (my $v = $drive->{"iops${dir}_max_length"}) {
1646             $opts .= ",throttling.iops$qmpname-max-length=$v";
1647         }
1648     }
1649
1650     if (my $serial = $drive->{serial}) {
1651         $serial = URI::Escape::uri_unescape($serial);
1652         $opts .= ",serial=$serial";
1653     }
1654
1655     $opts .= ",format=$format" if $format && !$drive->{format};
1656
1657     my $cache_direct = 0;
1658
1659     if (my $cache = $drive->{cache}) {
1660         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1661     } elsif (!drive_is_cdrom($drive)) {
1662         $opts .= ",cache=none";
1663         $cache_direct = 1;
1664     }
1665
1666     # aio native works only with O_DIRECT
1667     if (!$drive->{aio}) {
1668         if($cache_direct) {
1669             $opts .= ",aio=native";
1670         } else {
1671             $opts .= ",aio=threads";
1672         }
1673     }
1674
1675     if (!drive_is_cdrom($drive)) {
1676         my $detectzeroes;
1677         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1678             $detectzeroes = 'off';
1679         } elsif ($drive->{discard}) {
1680             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1681         } else {
1682             # This used to be our default with discard not being specified:
1683             $detectzeroes = 'on';
1684         }
1685         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1686     }
1687
1688     my $pathinfo = $path ? "file=$path," : '';
1689
1690     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1691 }
1692
1693 sub print_netdevice_full {
1694     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1695
1696     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1697
1698     my $device = $net->{model};
1699     if ($net->{model} eq 'virtio') {
1700          $device = 'virtio-net-pci';
1701      };
1702
1703     my $pciaddr = print_pci_addr("$netid", $bridges);
1704     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1705     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1706         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1707         my $vectors = $net->{queues} * 2 + 2;
1708         $tmpstr .= ",vectors=$vectors,mq=on";
1709     }
1710     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1711
1712     if ($use_old_bios_files) {
1713         my $romfile;
1714         if ($device eq 'virtio-net-pci') {
1715             $romfile = 'pxe-virtio.rom';
1716         } elsif ($device eq 'e1000') {
1717             $romfile = 'pxe-e1000.rom';
1718         } elsif ($device eq 'ne2k') {
1719             $romfile = 'pxe-ne2k_pci.rom';
1720         } elsif ($device eq 'pcnet') {
1721             $romfile = 'pxe-pcnet.rom';
1722         } elsif ($device eq 'rtl8139') {
1723             $romfile = 'pxe-rtl8139.rom';
1724         }
1725         $tmpstr .= ",romfile=$romfile" if $romfile;
1726     }
1727
1728     return $tmpstr;
1729 }
1730
1731 sub print_netdev_full {
1732     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1733
1734     my $i = '';
1735     if ($netid =~ m/^net(\d+)$/) {
1736         $i = int($1);
1737     }
1738
1739     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1740
1741     my $ifname = "tap${vmid}i$i";
1742
1743     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1744     die "interface name '$ifname' is too long (max 15 character)\n"
1745         if length($ifname) >= 16;
1746
1747     my $vhostparam = '';
1748     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1749
1750     my $vmname = $conf->{name} || "vm$vmid";
1751
1752     my $netdev = "";
1753     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1754
1755     if ($net->{bridge}) {
1756         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1757     } else {
1758         $netdev = "type=user,id=$netid,hostname=$vmname";
1759     }
1760
1761     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1762
1763     return $netdev;
1764 }
1765
1766
1767 sub print_cpu_device {
1768     my ($conf, $id) = @_;
1769
1770     my $kvm = $conf->{kvm} // 1;
1771     my $cpu = $kvm ? "kvm64" : "qemu64";
1772     if (my $cputype = $conf->{cpu}) {
1773         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1774             or die "Cannot parse cpu description: $cputype\n";
1775         $cpu = $cpuconf->{cputype};
1776     }
1777
1778     my $cores = $conf->{cores} || 1;
1779
1780     my $current_core = ($id - 1) % $cores;
1781     my $current_socket = int(($id - 1 - $current_core)/$cores);
1782
1783     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1784 }
1785
1786 sub drive_is_cdrom {
1787     my ($drive) = @_;
1788
1789     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1790
1791 }
1792
1793 sub parse_number_sets {
1794     my ($set) = @_;
1795     my $res = [];
1796     foreach my $part (split(/;/, $set)) {
1797         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1798             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1799             push @$res, [ $1, $2 ];
1800         } else {
1801             die "invalid range: $part\n";
1802         }
1803     }
1804     return $res;
1805 }
1806
1807 sub parse_numa {
1808     my ($data) = @_;
1809
1810     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1811     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1812     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1813     return $res;
1814 }
1815
1816 sub parse_hostpci {
1817     my ($value) = @_;
1818
1819     return undef if !$value;
1820
1821     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1822
1823     my @idlist = split(/;/, $res->{host});
1824     delete $res->{host};
1825     foreach my $id (@idlist) {
1826         if ($id =~ /^$PCIRE$/) {
1827             if (defined($2)) {
1828                 push @{$res->{pciid}}, { id => $1, function => $2 };
1829             } else {
1830                 my $pcidevices = lspci($1);
1831                 $res->{pciid} = $pcidevices->{$1};
1832             }
1833         } else {
1834             # should have been caught by parse_property_string already
1835             die "failed to parse PCI id: $id\n";
1836         }
1837     }
1838     return $res;
1839 }
1840
1841 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1842 sub parse_net {
1843     my ($data) = @_;
1844
1845     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1846     if ($@) {
1847         warn $@;
1848         return undef;
1849     }
1850     if (!defined($res->{macaddr})) {
1851         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1852         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1853     }
1854     return $res;
1855 }
1856
1857 sub print_net {
1858     my $net = shift;
1859
1860     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1861 }
1862
1863 sub add_random_macs {
1864     my ($settings) = @_;
1865
1866     foreach my $opt (keys %$settings) {
1867         next if $opt !~ m/^net(\d+)$/;
1868         my $net = parse_net($settings->{$opt});
1869         next if !$net;
1870         $settings->{$opt} = print_net($net);
1871     }
1872 }
1873
1874 sub vm_is_volid_owner {
1875     my ($storecfg, $vmid, $volid) = @_;
1876
1877     if ($volid !~  m|^/|) {
1878         my ($path, $owner);
1879         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1880         if ($owner && ($owner == $vmid)) {
1881             return 1;
1882         }
1883     }
1884
1885     return undef;
1886 }
1887
1888 sub split_flagged_list {
1889     my $text = shift || '';
1890     $text =~ s/[,;]/ /g;
1891     $text =~ s/^\s+//;
1892     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1893 }
1894
1895 sub join_flagged_list {
1896     my ($how, $lst) = @_;
1897     join $how, map { $lst->{$_} . $_ } keys %$lst;
1898 }
1899
1900 sub vmconfig_delete_pending_option {
1901     my ($conf, $key, $force) = @_;
1902
1903     delete $conf->{pending}->{$key};
1904     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1905     $pending_delete_hash->{$key} = $force ? '!' : '';
1906     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1907 }
1908
1909 sub vmconfig_undelete_pending_option {
1910     my ($conf, $key) = @_;
1911
1912     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1913     delete $pending_delete_hash->{$key};
1914
1915     if (%$pending_delete_hash) {
1916         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1917     } else {
1918         delete $conf->{pending}->{delete};
1919     }
1920 }
1921
1922 sub vmconfig_register_unused_drive {
1923     my ($storecfg, $vmid, $conf, $drive) = @_;
1924
1925     if (!drive_is_cdrom($drive)) {
1926         my $volid = $drive->{file};
1927         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1928             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1929         }
1930     }
1931 }
1932
1933 sub vmconfig_cleanup_pending {
1934     my ($conf) = @_;
1935
1936     # remove pending changes when nothing changed
1937     my $changes;
1938     foreach my $opt (keys %{$conf->{pending}}) {
1939         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1940             $changes = 1;
1941             delete $conf->{pending}->{$opt};
1942         }
1943     }
1944
1945     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1946     my $pending_delete_hash = {};
1947     while (my ($opt, $force) = each %$current_delete_hash) {
1948         if (defined($conf->{$opt})) {
1949             $pending_delete_hash->{$opt} = $force;
1950         } else {
1951             $changes = 1;
1952         }
1953     }
1954
1955     if (%$pending_delete_hash) {
1956         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1957     } else {
1958         delete $conf->{pending}->{delete};
1959     }
1960
1961     return $changes;
1962 }
1963
1964 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1965 my $smbios1_fmt = {
1966     uuid => {
1967         type => 'string',
1968         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1969         format_description => 'UUID',
1970         description => "Set SMBIOS1 UUID.",
1971         optional => 1,
1972     },
1973     version => {
1974         type => 'string',
1975         pattern => '\S+',
1976         format_description => 'string',
1977         description => "Set SMBIOS1 version.",
1978         optional => 1,
1979     },
1980     serial => {
1981         type => 'string',
1982         pattern => '\S+',
1983         format_description => 'string',
1984         description => "Set SMBIOS1 serial number.",
1985         optional => 1,
1986     },
1987     manufacturer => {
1988         type => 'string',
1989         pattern => '\S+',
1990         format_description => 'string',
1991         description => "Set SMBIOS1 manufacturer.",
1992         optional => 1,
1993     },
1994     product => {
1995         type => 'string',
1996         pattern => '\S+',
1997         format_description => 'string',
1998         description => "Set SMBIOS1 product ID.",
1999         optional => 1,
2000     },
2001     sku => {
2002         type => 'string',
2003         pattern => '\S+',
2004         format_description => 'string',
2005         description => "Set SMBIOS1 SKU string.",
2006         optional => 1,
2007     },
2008     family => {
2009         type => 'string',
2010         pattern => '\S+',
2011         format_description => 'string',
2012         description => "Set SMBIOS1 family string.",
2013         optional => 1,
2014     },
2015 };
2016
2017 sub parse_smbios1 {
2018     my ($data) = @_;
2019
2020     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2021     warn $@ if $@;
2022     return $res;
2023 }
2024
2025 sub print_smbios1 {
2026     my ($smbios1) = @_;
2027     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2028 }
2029
2030 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2031
2032 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2033 sub verify_bootdisk {
2034     my ($value, $noerr) = @_;
2035
2036     return $value if is_valid_drivename($value);
2037
2038     return undef if $noerr;
2039
2040     die "invalid boot disk '$value'\n";
2041 }
2042
2043 sub parse_watchdog {
2044     my ($value) = @_;
2045
2046     return undef if !$value;
2047
2048     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2049     warn $@ if $@;
2050     return $res;
2051 }
2052
2053 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2054 sub verify_usb_device {
2055     my ($value, $noerr) = @_;
2056
2057     return $value if parse_usb_device($value);
2058
2059     return undef if $noerr;
2060
2061     die "unable to parse usb device\n";
2062 }
2063
2064 # add JSON properties for create and set function
2065 sub json_config_properties {
2066     my $prop = shift;
2067
2068     foreach my $opt (keys %$confdesc) {
2069         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2070         $prop->{$opt} = $confdesc->{$opt};
2071     }
2072
2073     return $prop;
2074 }
2075
2076 sub check_type {
2077     my ($key, $value) = @_;
2078
2079     die "unknown setting '$key'\n" if !$confdesc->{$key};
2080
2081     my $type = $confdesc->{$key}->{type};
2082
2083     if (!defined($value)) {
2084         die "got undefined value\n";
2085     }
2086
2087     if ($value =~ m/[\n\r]/) {
2088         die "property contains a line feed\n";
2089     }
2090
2091     if ($type eq 'boolean') {
2092         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2093         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2094         die "type check ('boolean') failed - got '$value'\n";
2095     } elsif ($type eq 'integer') {
2096         return int($1) if $value =~ m/^(\d+)$/;
2097         die "type check ('integer') failed - got '$value'\n";
2098     } elsif ($type eq 'number') {
2099         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2100         die "type check ('number') failed - got '$value'\n";
2101     } elsif ($type eq 'string') {
2102         if (my $fmt = $confdesc->{$key}->{format}) {
2103             PVE::JSONSchema::check_format($fmt, $value);
2104             return $value;
2105         }
2106         $value =~ s/^\"(.*)\"$/$1/;
2107         return $value;
2108     } else {
2109         die "internal error"
2110     }
2111 }
2112
2113 sub check_iommu_support{
2114     #fixme : need to check IOMMU support
2115     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2116
2117     my $iommu=1;
2118     return $iommu;
2119
2120 }
2121
2122 sub touch_config {
2123     my ($vmid) = @_;
2124
2125     my $conf = PVE::QemuConfig->config_file($vmid);
2126     utime undef, undef, $conf;
2127 }
2128
2129 sub destroy_vm {
2130     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2131
2132     my $conffile = PVE::QemuConfig->config_file($vmid);
2133
2134     my $conf = PVE::QemuConfig->load_config($vmid);
2135
2136     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2137
2138     if ($conf->{template}) {
2139         # check if any base image is still used by a linked clone
2140         foreach_drive($conf, sub {
2141                 my ($ds, $drive) = @_;
2142
2143                 return if drive_is_cdrom($drive);
2144
2145                 my $volid = $drive->{file};
2146
2147                 return if !$volid || $volid =~ m|^/|;
2148
2149                 die "base volume '$volid' is still in use by linked cloned\n"
2150                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2151
2152         });
2153     }
2154
2155     # only remove disks owned by this VM
2156     foreach_drive($conf, sub {
2157         my ($ds, $drive) = @_;
2158
2159         return if drive_is_cdrom($drive);
2160
2161         my $volid = $drive->{file};
2162
2163         return if !$volid || $volid =~ m|^/|;
2164
2165         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2166         return if !$path || !$owner || ($owner != $vmid);
2167
2168         eval {
2169             PVE::Storage::vdisk_free($storecfg, $volid);
2170         };
2171         warn "Could not remove disk '$volid', check manually: $@" if $@;
2172
2173     });
2174
2175     if ($keep_empty_config) {
2176         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2177     } else {
2178         unlink $conffile;
2179     }
2180
2181     # also remove unused disk
2182     eval {
2183         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2184
2185         eval {
2186             PVE::Storage::foreach_volid($dl, sub {
2187                 my ($volid, $sid, $volname, $d) = @_;
2188                 PVE::Storage::vdisk_free($storecfg, $volid);
2189             });
2190         };
2191         warn $@ if $@;
2192
2193     };
2194     warn $@ if $@;
2195 }
2196
2197 sub parse_vm_config {
2198     my ($filename, $raw) = @_;
2199
2200     return undef if !defined($raw);
2201
2202     my $res = {
2203         digest => Digest::SHA::sha1_hex($raw),
2204         snapshots => {},
2205         pending => {},
2206     };
2207
2208     $filename =~ m|/qemu-server/(\d+)\.conf$|
2209         || die "got strange filename '$filename'";
2210
2211     my $vmid = $1;
2212
2213     my $conf = $res;
2214     my $descr;
2215     my $section = '';
2216
2217     my @lines = split(/\n/, $raw);
2218     foreach my $line (@lines) {
2219         next if $line =~ m/^\s*$/;
2220
2221         if ($line =~ m/^\[PENDING\]\s*$/i) {
2222             $section = 'pending';
2223             if (defined($descr)) {
2224                 $descr =~ s/\s+$//;
2225                 $conf->{description} = $descr;
2226             }
2227             $descr = undef;
2228             $conf = $res->{$section} = {};
2229             next;
2230
2231         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2232             $section = $1;
2233             if (defined($descr)) {
2234                 $descr =~ s/\s+$//;
2235                 $conf->{description} = $descr;
2236             }
2237             $descr = undef;
2238             $conf = $res->{snapshots}->{$section} = {};
2239             next;
2240         }
2241
2242         if ($line =~ m/^\#(.*)\s*$/) {
2243             $descr = '' if !defined($descr);
2244             $descr .= PVE::Tools::decode_text($1) . "\n";
2245             next;
2246         }
2247
2248         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2249             $descr = '' if !defined($descr);
2250             $descr .= PVE::Tools::decode_text($2);
2251         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2252             $conf->{snapstate} = $1;
2253         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2254             my $key = $1;
2255             my $value = $2;
2256             $conf->{$key} = $value;
2257         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2258             my $value = $1;
2259             if ($section eq 'pending') {
2260                 $conf->{delete} = $value; # we parse this later
2261             } else {
2262                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2263             }
2264         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2265             my $key = $1;
2266             my $value = $2;
2267             eval { $value = check_type($key, $value); };
2268             if ($@) {
2269                 warn "vm $vmid - unable to parse value of '$key' - $@";
2270             } else {
2271                 $key = 'ide2' if $key eq 'cdrom';
2272                 my $fmt = $confdesc->{$key}->{format};
2273                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2274                     my $v = parse_drive($key, $value);
2275                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2276                         $v->{file} = $volid;
2277                         $value = print_drive($vmid, $v);
2278                     } else {
2279                         warn "vm $vmid - unable to parse value of '$key'\n";
2280                         next;
2281                     }
2282                 }
2283
2284                 $conf->{$key} = $value;
2285             }
2286         }
2287     }
2288
2289     if (defined($descr)) {
2290         $descr =~ s/\s+$//;
2291         $conf->{description} = $descr;
2292     }
2293     delete $res->{snapstate}; # just to be sure
2294
2295     return $res;
2296 }
2297
2298 sub write_vm_config {
2299     my ($filename, $conf) = @_;
2300
2301     delete $conf->{snapstate}; # just to be sure
2302
2303     if ($conf->{cdrom}) {
2304         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2305         $conf->{ide2} = $conf->{cdrom};
2306         delete $conf->{cdrom};
2307     }
2308
2309     # we do not use 'smp' any longer
2310     if ($conf->{sockets}) {
2311         delete $conf->{smp};
2312     } elsif ($conf->{smp}) {
2313         $conf->{sockets} = $conf->{smp};
2314         delete $conf->{cores};
2315         delete $conf->{smp};
2316     }
2317
2318     my $used_volids = {};
2319
2320     my $cleanup_config = sub {
2321         my ($cref, $pending, $snapname) = @_;
2322
2323         foreach my $key (keys %$cref) {
2324             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2325                 $key eq 'snapstate' || $key eq 'pending';
2326             my $value = $cref->{$key};
2327             if ($key eq 'delete') {
2328                 die "propertry 'delete' is only allowed in [PENDING]\n"
2329                     if !$pending;
2330                 # fixme: check syntax?
2331                 next;
2332             }
2333             eval { $value = check_type($key, $value); };
2334             die "unable to parse value of '$key' - $@" if $@;
2335
2336             $cref->{$key} = $value;
2337
2338             if (!$snapname && is_valid_drivename($key)) {
2339                 my $drive = parse_drive($key, $value);
2340                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2341             }
2342         }
2343     };
2344
2345     &$cleanup_config($conf);
2346
2347     &$cleanup_config($conf->{pending}, 1);
2348
2349     foreach my $snapname (keys %{$conf->{snapshots}}) {
2350         die "internal error" if $snapname eq 'pending';
2351         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2352     }
2353
2354     # remove 'unusedX' settings if we re-add a volume
2355     foreach my $key (keys %$conf) {
2356         my $value = $conf->{$key};
2357         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2358             delete $conf->{$key};
2359         }
2360     }
2361
2362     my $generate_raw_config = sub {
2363         my ($conf, $pending) = @_;
2364
2365         my $raw = '';
2366
2367         # add description as comment to top of file
2368         if (defined(my $descr = $conf->{description})) {
2369             if ($descr) {
2370                 foreach my $cl (split(/\n/, $descr)) {
2371                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2372                 }
2373             } else {
2374                 $raw .= "#\n" if $pending;
2375             }
2376         }
2377
2378         foreach my $key (sort keys %$conf) {
2379             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2380             $raw .= "$key: $conf->{$key}\n";
2381         }
2382         return $raw;
2383     };
2384
2385     my $raw = &$generate_raw_config($conf);
2386
2387     if (scalar(keys %{$conf->{pending}})){
2388         $raw .= "\n[PENDING]\n";
2389         $raw .= &$generate_raw_config($conf->{pending}, 1);
2390     }
2391
2392     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2393         $raw .= "\n[$snapname]\n";
2394         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2395     }
2396
2397     return $raw;
2398 }
2399
2400 sub load_defaults {
2401
2402     my $res = {};
2403
2404     # we use static defaults from our JSON schema configuration
2405     foreach my $key (keys %$confdesc) {
2406         if (defined(my $default = $confdesc->{$key}->{default})) {
2407             $res->{$key} = $default;
2408         }
2409     }
2410
2411     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2412     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2413
2414     return $res;
2415 }
2416
2417 sub config_list {
2418     my $vmlist = PVE::Cluster::get_vmlist();
2419     my $res = {};
2420     return $res if !$vmlist || !$vmlist->{ids};
2421     my $ids = $vmlist->{ids};
2422
2423     foreach my $vmid (keys %$ids) {
2424         my $d = $ids->{$vmid};
2425         next if !$d->{node} || $d->{node} ne $nodename;
2426         next if !$d->{type} || $d->{type} ne 'qemu';
2427         $res->{$vmid}->{exists} = 1;
2428     }
2429     return $res;
2430 }
2431
2432 # test if VM uses local resources (to prevent migration)
2433 sub check_local_resources {
2434     my ($conf, $noerr) = @_;
2435
2436     my $loc_res = 0;
2437
2438     $loc_res = 1 if $conf->{hostusb}; # old syntax
2439     $loc_res = 1 if $conf->{hostpci}; # old syntax
2440
2441     foreach my $k (keys %$conf) {
2442         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2443         # sockets are safe: they will recreated be on the target side post-migrate
2444         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2445         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2446     }
2447
2448     die "VM uses local resources\n" if $loc_res && !$noerr;
2449
2450     return $loc_res;
2451 }
2452
2453 # check if used storages are available on all nodes (use by migrate)
2454 sub check_storage_availability {
2455     my ($storecfg, $conf, $node) = @_;
2456
2457     foreach_drive($conf, sub {
2458         my ($ds, $drive) = @_;
2459
2460         my $volid = $drive->{file};
2461         return if !$volid;
2462
2463         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2464         return if !$sid;
2465
2466         # check if storage is available on both nodes
2467         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2468         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2469    });
2470 }
2471
2472 # list nodes where all VM images are available (used by has_feature API)
2473 sub shared_nodes {
2474     my ($conf, $storecfg) = @_;
2475
2476     my $nodelist = PVE::Cluster::get_nodelist();
2477     my $nodehash = { map { $_ => 1 } @$nodelist };
2478     my $nodename = PVE::INotify::nodename();
2479
2480     foreach_drive($conf, sub {
2481         my ($ds, $drive) = @_;
2482
2483         my $volid = $drive->{file};
2484         return if !$volid;
2485
2486         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2487         if ($storeid) {
2488             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2489             if ($scfg->{disable}) {
2490                 $nodehash = {};
2491             } elsif (my $avail = $scfg->{nodes}) {
2492                 foreach my $node (keys %$nodehash) {
2493                     delete $nodehash->{$node} if !$avail->{$node};
2494                 }
2495             } elsif (!$scfg->{shared}) {
2496                 foreach my $node (keys %$nodehash) {
2497                     delete $nodehash->{$node} if $node ne $nodename
2498                 }
2499             }
2500         }
2501     });
2502
2503     return $nodehash
2504 }
2505
2506 sub check_cmdline {
2507     my ($pidfile, $pid) = @_;
2508
2509     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2510     if (defined($fh)) {
2511         my $line = <$fh>;
2512         $fh->close;
2513         return undef if !$line;
2514         my @param = split(/\0/, $line);
2515
2516         my $cmd = $param[0];
2517         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2518
2519         for (my $i = 0; $i < scalar (@param); $i++) {
2520             my $p = $param[$i];
2521             next if !$p;
2522             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2523                 my $p = $param[$i+1];
2524                 return 1 if $p && ($p eq $pidfile);
2525                 return undef;
2526             }
2527         }
2528     }
2529     return undef;
2530 }
2531
2532 sub check_running {
2533     my ($vmid, $nocheck, $node) = @_;
2534
2535     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2536
2537     die "unable to find configuration file for VM $vmid - no such machine\n"
2538         if !$nocheck && ! -f $filename;
2539
2540     my $pidfile = pidfile_name($vmid);
2541
2542     if (my $fd = IO::File->new("<$pidfile")) {
2543         my $st = stat($fd);
2544         my $line = <$fd>;
2545         close($fd);
2546
2547         my $mtime = $st->mtime;
2548         if ($mtime > time()) {
2549             warn "file '$filename' modified in future\n";
2550         }
2551
2552         if ($line =~ m/^(\d+)$/) {
2553             my $pid = $1;
2554             if (check_cmdline($pidfile, $pid)) {
2555                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2556                     return $pid;
2557                 }
2558             }
2559         }
2560     }
2561
2562     return undef;
2563 }
2564
2565 sub vzlist {
2566
2567     my $vzlist = config_list();
2568
2569     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2570
2571     while (defined(my $de = $fd->read)) {
2572         next if $de !~ m/^(\d+)\.pid$/;
2573         my $vmid = $1;
2574         next if !defined($vzlist->{$vmid});
2575         if (my $pid = check_running($vmid)) {
2576             $vzlist->{$vmid}->{pid} = $pid;
2577         }
2578     }
2579
2580     return $vzlist;
2581 }
2582
2583 sub disksize {
2584     my ($storecfg, $conf) = @_;
2585
2586     my $bootdisk = $conf->{bootdisk};
2587     return undef if !$bootdisk;
2588     return undef if !is_valid_drivename($bootdisk);
2589
2590     return undef if !$conf->{$bootdisk};
2591
2592     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2593     return undef if !defined($drive);
2594
2595     return undef if drive_is_cdrom($drive);
2596
2597     my $volid = $drive->{file};
2598     return undef if !$volid;
2599
2600     return $drive->{size};
2601 }
2602
2603 my $last_proc_pid_stat;
2604
2605 # get VM status information
2606 # This must be fast and should not block ($full == false)
2607 # We only query KVM using QMP if $full == true (this can be slow)
2608 sub vmstatus {
2609     my ($opt_vmid, $full) = @_;
2610
2611     my $res = {};
2612
2613     my $storecfg = PVE::Storage::config();
2614
2615     my $list = vzlist();
2616     my $defaults = load_defaults();
2617
2618     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2619
2620     my $cpucount = $cpuinfo->{cpus} || 1;
2621
2622     foreach my $vmid (keys %$list) {
2623         next if $opt_vmid && ($vmid ne $opt_vmid);
2624
2625         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2626         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2627
2628         my $d = {};
2629         $d->{pid} = $list->{$vmid}->{pid};
2630
2631         # fixme: better status?
2632         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2633
2634         my $size = disksize($storecfg, $conf);
2635         if (defined($size)) {
2636             $d->{disk} = 0; # no info available
2637             $d->{maxdisk} = $size;
2638         } else {
2639             $d->{disk} = 0;
2640             $d->{maxdisk} = 0;
2641         }
2642
2643         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2644             * ($conf->{cores} || $defaults->{cores});
2645         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2646         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2647
2648         $d->{name} = $conf->{name} || "VM $vmid";
2649         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2650             : $defaults->{memory}*(1024*1024);
2651
2652         if ($conf->{balloon}) {
2653             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2654             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2655                 : $defaults->{shares};
2656         }
2657
2658         $d->{uptime} = 0;
2659         $d->{cpu} = 0;
2660         $d->{mem} = 0;
2661
2662         $d->{netout} = 0;
2663         $d->{netin} = 0;
2664
2665         $d->{diskread} = 0;
2666         $d->{diskwrite} = 0;
2667
2668         $d->{template} = PVE::QemuConfig->is_template($conf);
2669
2670         $res->{$vmid} = $d;
2671     }
2672
2673     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2674     foreach my $dev (keys %$netdev) {
2675         next if $dev !~ m/^tap([1-9]\d*)i/;
2676         my $vmid = $1;
2677         my $d = $res->{$vmid};
2678         next if !$d;
2679
2680         $d->{netout} += $netdev->{$dev}->{receive};
2681         $d->{netin} += $netdev->{$dev}->{transmit};
2682
2683         if ($full) {
2684             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2685             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2686         }
2687
2688     }
2689
2690     my $ctime = gettimeofday;
2691
2692     foreach my $vmid (keys %$list) {
2693
2694         my $d = $res->{$vmid};
2695         my $pid = $d->{pid};
2696         next if !$pid;
2697
2698         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2699         next if !$pstat; # not running
2700
2701         my $used = $pstat->{utime} + $pstat->{stime};
2702
2703         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2704
2705         if ($pstat->{vsize}) {
2706             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2707         }
2708
2709         my $old = $last_proc_pid_stat->{$pid};
2710         if (!$old) {
2711             $last_proc_pid_stat->{$pid} = {
2712                 time => $ctime,
2713                 used => $used,
2714                 cpu => 0,
2715             };
2716             next;
2717         }
2718
2719         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2720
2721         if ($dtime > 1000) {
2722             my $dutime = $used -  $old->{used};
2723
2724             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2725             $last_proc_pid_stat->{$pid} = {
2726                 time => $ctime,
2727                 used => $used,
2728                 cpu => $d->{cpu},
2729             };
2730         } else {
2731             $d->{cpu} = $old->{cpu};
2732         }
2733     }
2734
2735     return $res if !$full;
2736
2737     my $qmpclient = PVE::QMPClient->new();
2738
2739     my $ballooncb = sub {
2740         my ($vmid, $resp) = @_;
2741
2742         my $info = $resp->{'return'};
2743         return if !$info->{max_mem};
2744
2745         my $d = $res->{$vmid};
2746
2747         # use memory assigned to VM
2748         $d->{maxmem} = $info->{max_mem};
2749         $d->{balloon} = $info->{actual};
2750
2751         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2752             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2753             $d->{freemem} = $info->{free_mem};
2754         }
2755
2756         $d->{ballooninfo} = $info;
2757     };
2758
2759     my $blockstatscb = sub {
2760         my ($vmid, $resp) = @_;
2761         my $data = $resp->{'return'} || [];
2762         my $totalrdbytes = 0;
2763         my $totalwrbytes = 0;
2764
2765         for my $blockstat (@$data) {
2766             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2767             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2768
2769             $blockstat->{device} =~ s/drive-//;
2770             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2771         }
2772         $res->{$vmid}->{diskread} = $totalrdbytes;
2773         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2774     };
2775
2776     my $statuscb = sub {
2777         my ($vmid, $resp) = @_;
2778
2779         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2780         # this fails if ballon driver is not loaded, so this must be
2781         # the last commnand (following command are aborted if this fails).
2782         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2783
2784         my $status = 'unknown';
2785         if (!defined($status = $resp->{'return'}->{status})) {
2786             warn "unable to get VM status\n";
2787             return;
2788         }
2789
2790         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2791     };
2792
2793     foreach my $vmid (keys %$list) {
2794         next if $opt_vmid && ($vmid ne $opt_vmid);
2795         next if !$res->{$vmid}->{pid}; # not running
2796         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2797     }
2798
2799     $qmpclient->queue_execute(undef, 2);
2800
2801     foreach my $vmid (keys %$list) {
2802         next if $opt_vmid && ($vmid ne $opt_vmid);
2803         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2804     }
2805
2806     return $res;
2807 }
2808
2809 sub foreach_drive {
2810     my ($conf, $func, @param) = @_;
2811
2812     foreach my $ds (valid_drive_names()) {
2813         next if !defined($conf->{$ds});
2814
2815         my $drive = parse_drive($ds, $conf->{$ds});
2816         next if !$drive;
2817
2818         &$func($ds, $drive, @param);
2819     }
2820 }
2821
2822 sub foreach_volid {
2823     my ($conf, $func, @param) = @_;
2824
2825     my $volhash = {};
2826
2827     my $test_volid = sub {
2828         my ($volid, $is_cdrom, $replicate, $snapname) = @_;
2829
2830         return if !$volid;
2831
2832         $volhash->{$volid}->{cdrom} //= 1;
2833         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2834
2835         $volhash->{$volid}->{replicate} //= 0;
2836         $volhash->{$volid}->{replicate} = 1 if $replicate;
2837
2838         $volhash->{$volid}->{referenced_in_config} //= 0;
2839         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
2840
2841         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
2842             if defined($snapname);
2843     };
2844
2845     foreach_drive($conf, sub {
2846         my ($ds, $drive) = @_;
2847         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, undef);
2848     });
2849
2850     foreach my $snapname (keys %{$conf->{snapshots}}) {
2851         my $snap = $conf->{snapshots}->{$snapname};
2852         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
2853         foreach_drive($snap, sub {
2854             my ($ds, $drive) = @_;
2855             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $snapname);
2856         });
2857     }
2858
2859     foreach my $volid (keys %$volhash) {
2860         &$func($volid, $volhash->{$volid}, @param);
2861     }
2862 }
2863
2864 sub vga_conf_has_spice {
2865     my ($vga) = @_;
2866
2867     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2868
2869     return $1 || 1;
2870 }
2871
2872 sub config_to_command {
2873     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2874
2875     my $cmd = [];
2876     my $globalFlags = [];
2877     my $machineFlags = [];
2878     my $rtcFlags = [];
2879     my $cpuFlags = [];
2880     my $devices = [];
2881     my $pciaddr = '';
2882     my $bridges = {};
2883     my $kvmver = kvm_user_version();
2884     my $vernum = 0; # unknown
2885     my $ostype = $conf->{ostype};
2886     my $winversion = windows_version($ostype);
2887     my $kvm = $conf->{kvm} // 1;
2888
2889     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
2890
2891     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2892         $vernum = $1*1000000+$2*1000;
2893     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2894         $vernum = $1*1000000+$2*1000+$3;
2895     }
2896
2897     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2898
2899     my $have_ovz = -f '/proc/vz/vestat';
2900
2901     my $q35 = machine_type_is_q35($conf);
2902     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2903     my $machine_type = $forcemachine || $conf->{machine};
2904     my $use_old_bios_files = undef;
2905     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2906
2907     my $cpuunits = defined($conf->{cpuunits}) ?
2908             $conf->{cpuunits} : $defaults->{cpuunits};
2909
2910     push @$cmd, '/usr/bin/kvm';
2911
2912     push @$cmd, '-id', $vmid;
2913
2914     my $use_virtio = 0;
2915
2916     my $qmpsocket = qmp_socket($vmid);
2917     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2918     push @$cmd, '-mon', "chardev=qmp,mode=control";
2919
2920
2921     push @$cmd, '-pidfile' , pidfile_name($vmid);
2922
2923     push @$cmd, '-daemonize';
2924
2925     if ($conf->{smbios1}) {
2926         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2927     }
2928
2929     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2930         die "uefi base image not found\n" if ! -f $OVMF_CODE;
2931
2932         my $path;
2933         my $format;
2934         if (my $efidisk = $conf->{efidisk0}) {
2935             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
2936             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2937             $format = $d->{format};
2938             if ($storeid) {
2939                 $path = PVE::Storage::path($storecfg, $d->{file});
2940                 if (!defined($format)) {
2941                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2942                     $format = qemu_img_format($scfg, $volname);
2943                 }
2944             } else {
2945                 $path = $d->{file};
2946                 die "efidisk format must be specified\n"
2947                     if !defined($format);
2948             }
2949         } else {
2950             warn "no efidisk configured! Using temporary efivars disk.\n";
2951             $path = "/tmp/$vmid-ovmf.fd";
2952             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
2953             $format = 'raw';
2954         }
2955
2956         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
2957         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
2958     }
2959
2960
2961     # add usb controllers
2962     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2963     push @$devices, @usbcontrollers if @usbcontrollers;
2964     my $vga = $conf->{vga};
2965
2966     my $qxlnum = vga_conf_has_spice($vga);
2967     $vga = 'qxl' if $qxlnum;
2968
2969     if (!$vga) {
2970         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
2971             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
2972         } else {
2973             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
2974         }
2975     }
2976
2977     # enable absolute mouse coordinates (needed by vnc)
2978     my $tablet;
2979     if (defined($conf->{tablet})) {
2980         $tablet = $conf->{tablet};
2981     } else {
2982         $tablet = $defaults->{tablet};
2983         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2984         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2985     }
2986
2987     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2988
2989     my $kvm_off = 0;
2990     my $gpu_passthrough;
2991
2992     # host pci devices
2993     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2994         my $d = parse_hostpci($conf->{"hostpci$i"});
2995         next if !$d;
2996
2997         my $pcie = $d->{pcie};
2998         if($pcie){
2999             die "q35 machine model is not enabled" if !$q35;
3000             $pciaddr = print_pcie_addr("hostpci$i");
3001         }else{
3002             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3003         }
3004
3005         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3006         my $romfile = $d->{romfile};
3007
3008         my $xvga = '';
3009         if ($d->{'x-vga'}) {
3010             $xvga = ',x-vga=on';
3011             $kvm_off = 1;
3012             $vga = 'none';
3013             $gpu_passthrough = 1;
3014
3015             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3016                 $xvga = "";
3017             }
3018         }
3019         my $pcidevices = $d->{pciid};
3020         my $multifunction = 1 if @$pcidevices > 1;
3021
3022         my $j=0;
3023         foreach my $pcidevice (@$pcidevices) {
3024
3025             my $id = "hostpci$i";
3026             $id .= ".$j" if $multifunction;
3027             my $addr = $pciaddr;
3028             $addr .= ".$j" if $multifunction;
3029             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3030
3031             if($j == 0){
3032                 $devicestr .= "$rombar$xvga";
3033                 $devicestr .= ",multifunction=on" if $multifunction;
3034                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3035             }
3036
3037             push @$devices, '-device', $devicestr;
3038             $j++;
3039         }
3040     }
3041
3042     # usb devices
3043     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3044     push @$devices, @usbdevices if @usbdevices;
3045     # serial devices
3046     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3047         if (my $path = $conf->{"serial$i"}) {
3048             if ($path eq 'socket') {
3049                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3050                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3051                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3052             } else {
3053                 die "no such serial device\n" if ! -c $path;
3054                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3055                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3056             }
3057         }
3058     }
3059
3060     # parallel devices
3061     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3062         if (my $path = $conf->{"parallel$i"}) {
3063             die "no such parallel device\n" if ! -c $path;
3064             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3065             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3066             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3067         }
3068     }
3069
3070     my $vmname = $conf->{name} || "vm$vmid";
3071
3072     push @$cmd, '-name', $vmname;
3073
3074     my $sockets = 1;
3075     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3076     $sockets = $conf->{sockets} if  $conf->{sockets};
3077
3078     my $cores = $conf->{cores} || 1;
3079
3080     my $maxcpus = $sockets * $cores;
3081
3082     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3083
3084     my $allowed_vcpus = $cpuinfo->{cpus};
3085
3086     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3087         if ($allowed_vcpus < $maxcpus);
3088
3089     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3090
3091         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3092         for (my $i = 2; $i <= $vcpus; $i++)  {
3093             my $cpustr = print_cpu_device($conf,$i);
3094             push @$cmd, '-device', $cpustr;
3095         }
3096
3097     } else {
3098
3099         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3100     }
3101     push @$cmd, '-nodefaults';
3102
3103     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3104
3105     my $bootindex_hash = {};
3106     my $i = 1;
3107     foreach my $o (split(//, $bootorder)) {
3108         $bootindex_hash->{$o} = $i*100;
3109         $i++;
3110     }
3111
3112     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3113
3114     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3115
3116     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3117
3118     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3119
3120     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3121         my $socket = vnc_socket($vmid);
3122         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3123     } else {
3124         push @$cmd, '-nographic';
3125     }
3126
3127     # time drift fix
3128     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3129
3130     my $useLocaltime = $conf->{localtime};
3131
3132     if ($winversion >= 5) { # windows
3133         $useLocaltime = 1 if !defined($conf->{localtime});
3134
3135         # use time drift fix when acpi is enabled
3136         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3137             $tdf = 1 if !defined($conf->{tdf});
3138         }
3139     }
3140
3141     if ($winversion >= 6) {
3142         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3143         push @$cmd, '-no-hpet';
3144     }
3145
3146     push @$rtcFlags, 'driftfix=slew' if $tdf;
3147
3148     if (!$kvm) {
3149         push @$machineFlags, 'accel=tcg';
3150     }
3151
3152     if ($machine_type) {
3153         push @$machineFlags, "type=${machine_type}";
3154     }
3155
3156     if ($conf->{startdate}) {
3157         push @$rtcFlags, "base=$conf->{startdate}";
3158     } elsif ($useLocaltime) {
3159         push @$rtcFlags, 'base=localtime';
3160     }
3161
3162     my $cpu = $kvm ? "kvm64" : "qemu64";
3163     if (my $cputype = $conf->{cpu}) {
3164         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3165             or die "Cannot parse cpu description: $cputype\n";
3166         $cpu = $cpuconf->{cputype};
3167         $kvm_off = 1 if $cpuconf->{hidden};
3168
3169         if (defined(my $flags = $cpuconf->{flags})) {
3170             push @$cpuFlags, $flags;
3171         }
3172     }
3173
3174     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3175
3176     push @$cpuFlags , '-x2apic'
3177         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3178
3179     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3180
3181     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3182
3183     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3184
3185         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3186         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3187     }
3188
3189     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3190
3191     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3192
3193     push @$cpuFlags, 'kvm=off' if $kvm_off;
3194
3195     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3196         die "internal error"; # should not happen
3197
3198     push @$cpuFlags, "vendor=${cpu_vendor}"
3199         if $cpu_vendor ne 'default';
3200
3201     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3202
3203     push @$cmd, '-cpu', $cpu;
3204
3205     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3206
3207     push @$cmd, '-S' if $conf->{freeze};
3208
3209     # set keyboard layout
3210     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3211     push @$cmd, '-k', $kb if $kb;
3212
3213     # enable sound
3214     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3215     #push @$cmd, '-soundhw', 'es1370';
3216     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3217
3218     if($conf->{agent}) {
3219         my $qgasocket = qmp_socket($vmid, 1);
3220         my $pciaddr = print_pci_addr("qga0", $bridges);
3221         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3222         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3223         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3224     }
3225
3226     my $spice_port;
3227
3228     if ($qxlnum) {
3229         if ($qxlnum > 1) {
3230             if ($winversion){
3231                 for(my $i = 1; $i < $qxlnum; $i++){
3232                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3233                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3234                 }
3235             } else {
3236                 # assume other OS works like Linux
3237                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3238                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3239             }
3240         }
3241
3242         my $pciaddr = print_pci_addr("spice", $bridges);
3243
3244         my $nodename = PVE::INotify::nodename();
3245         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3246         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3247         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3248         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3249         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3250
3251         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3252
3253         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3254         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3255         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3256     }
3257
3258     # enable balloon by default, unless explicitly disabled
3259     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3260         $pciaddr = print_pci_addr("balloon0", $bridges);
3261         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3262     }
3263
3264     if ($conf->{watchdog}) {
3265         my $wdopts = parse_watchdog($conf->{watchdog});
3266         $pciaddr = print_pci_addr("watchdog", $bridges);
3267         my $watchdog = $wdopts->{model} || 'i6300esb';
3268         push @$devices, '-device', "$watchdog$pciaddr";
3269         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3270     }
3271
3272     my $vollist = [];
3273     my $scsicontroller = {};
3274     my $ahcicontroller = {};
3275     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3276
3277     # Add iscsi initiator name if available
3278     if (my $initiator = get_initiator_name()) {
3279         push @$devices, '-iscsi', "initiator-name=$initiator";
3280     }
3281
3282     foreach_drive($conf, sub {
3283         my ($ds, $drive) = @_;
3284
3285         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3286             push @$vollist, $drive->{file};
3287         }
3288
3289         # ignore efidisk here, already added in bios/fw handling code above
3290         return if $drive->{interface} eq 'efidisk';
3291
3292         $use_virtio = 1 if $ds =~ m/^virtio/;
3293
3294         if (drive_is_cdrom ($drive)) {
3295             if ($bootindex_hash->{d}) {
3296                 $drive->{bootindex} = $bootindex_hash->{d};
3297                 $bootindex_hash->{d} += 1;
3298             }
3299         } else {
3300             if ($bootindex_hash->{c}) {
3301                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3302                 $bootindex_hash->{c} += 1;
3303             }
3304         }
3305
3306         if($drive->{interface} eq 'virtio'){
3307            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3308         }
3309
3310         if ($drive->{interface} eq 'scsi') {
3311
3312             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3313
3314             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3315             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3316
3317             my $iothread = '';
3318             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3319                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3320                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3321             } elsif ($drive->{iothread}) {
3322                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3323             }
3324
3325             my $queues = '';
3326             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3327                 $queues = ",num_queues=$drive->{queues}";
3328             }
3329
3330             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3331             $scsicontroller->{$controller}=1;
3332         }
3333
3334         if ($drive->{interface} eq 'sata') {
3335            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3336            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3337            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3338            $ahcicontroller->{$controller}=1;
3339         }
3340
3341         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3342         push @$devices, '-drive',$drive_cmd;
3343         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3344     });
3345
3346     for (my $i = 0; $i < $MAX_NETS; $i++) {
3347          next if !$conf->{"net$i"};
3348          my $d = parse_net($conf->{"net$i"});
3349          next if !$d;
3350
3351          $use_virtio = 1 if $d->{model} eq 'virtio';
3352
3353          if ($bootindex_hash->{n}) {
3354             $d->{bootindex} = $bootindex_hash->{n};
3355             $bootindex_hash->{n} += 1;
3356          }
3357
3358          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3359          push @$devices, '-netdev', $netdevfull;
3360
3361          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3362          push @$devices, '-device', $netdevicefull;
3363     }
3364
3365     if (!$q35) {
3366         # add pci bridges
3367         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3368            $bridges->{1} = 1;
3369            $bridges->{2} = 1;
3370         }
3371
3372         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3373
3374         while (my ($k, $v) = each %$bridges) {
3375             $pciaddr = print_pci_addr("pci.$k");
3376             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3377         }
3378     }
3379
3380     # add custom args
3381     if ($conf->{args}) {
3382         my $aa = PVE::Tools::split_args($conf->{args});
3383         push @$cmd, @$aa;
3384     }
3385
3386     push @$cmd, @$devices;
3387     push @$cmd, '-rtc', join(',', @$rtcFlags)
3388         if scalar(@$rtcFlags);
3389     push @$cmd, '-machine', join(',', @$machineFlags)
3390         if scalar(@$machineFlags);
3391     push @$cmd, '-global', join(',', @$globalFlags)
3392         if scalar(@$globalFlags);
3393
3394     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3395 }
3396
3397 sub vnc_socket {
3398     my ($vmid) = @_;
3399     return "${var_run_tmpdir}/$vmid.vnc";
3400 }
3401
3402 sub spice_port {
3403     my ($vmid) = @_;
3404
3405     my $res = vm_mon_cmd($vmid, 'query-spice');
3406
3407     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3408 }
3409
3410 sub qmp_socket {
3411     my ($vmid, $qga) = @_;
3412     my $sockettype = $qga ? 'qga' : 'qmp';
3413     return "${var_run_tmpdir}/$vmid.$sockettype";
3414 }
3415
3416 sub pidfile_name {
3417     my ($vmid) = @_;
3418     return "${var_run_tmpdir}/$vmid.pid";
3419 }
3420
3421 sub vm_devices_list {
3422     my ($vmid) = @_;
3423
3424     my $res = vm_mon_cmd($vmid, 'query-pci');
3425     my $devices = {};
3426     foreach my $pcibus (@$res) {
3427         foreach my $device (@{$pcibus->{devices}}) {
3428             next if !$device->{'qdev_id'};
3429             if ($device->{'pci_bridge'}) {
3430                 $devices->{$device->{'qdev_id'}} = 1;
3431                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3432                     next if !$bridge_device->{'qdev_id'};
3433                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3434                     $devices->{$device->{'qdev_id'}}++;
3435                 }
3436             } else {
3437                 $devices->{$device->{'qdev_id'}} = 1;
3438             }
3439         }
3440     }
3441
3442     my $resblock = vm_mon_cmd($vmid, 'query-block');
3443     foreach my $block (@$resblock) {
3444         if($block->{device} =~ m/^drive-(\S+)/){
3445                 $devices->{$1} = 1;
3446         }
3447     }
3448
3449     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3450     foreach my $mice (@$resmice) {
3451         if ($mice->{name} eq 'QEMU HID Tablet') {
3452             $devices->{tablet} = 1;
3453             last;
3454         }
3455     }
3456
3457     # for usb devices there is no query-usb
3458     # but we can iterate over the entries in
3459     # qom-list path=/machine/peripheral
3460     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3461     foreach my $per (@$resperipheral) {
3462         if ($per->{name} =~ m/^usb\d+$/) {
3463             $devices->{$per->{name}} = 1;
3464         }
3465     }
3466
3467     return $devices;
3468 }
3469
3470 sub vm_deviceplug {
3471     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3472
3473     my $q35 = machine_type_is_q35($conf);
3474
3475     my $devices_list = vm_devices_list($vmid);
3476     return 1 if defined($devices_list->{$deviceid});
3477
3478     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3479
3480     if ($deviceid eq 'tablet') {
3481
3482         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3483
3484     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3485
3486         die "usb hotplug currently not reliable\n";
3487         # since we can't reliably hot unplug all added usb devices
3488         # and usb passthrough disables live migration
3489         # we disable usb hotplugging for now
3490         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3491
3492     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3493
3494         qemu_iothread_add($vmid, $deviceid, $device);
3495
3496         qemu_driveadd($storecfg, $vmid, $device);
3497         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3498
3499         qemu_deviceadd($vmid, $devicefull);
3500         eval { qemu_deviceaddverify($vmid, $deviceid); };
3501         if (my $err = $@) {
3502             eval { qemu_drivedel($vmid, $deviceid); };
3503             warn $@ if $@;
3504             die $err;
3505         }
3506
3507     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3508
3509
3510         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3511         my $pciaddr = print_pci_addr($deviceid);
3512         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3513
3514         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3515
3516         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3517             qemu_iothread_add($vmid, $deviceid, $device);
3518             $devicefull .= ",iothread=iothread-$deviceid";
3519         }
3520
3521         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3522             $devicefull .= ",num_queues=$device->{queues}";
3523         }
3524
3525         qemu_deviceadd($vmid, $devicefull);
3526         qemu_deviceaddverify($vmid, $deviceid);
3527
3528     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3529
3530         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3531         qemu_driveadd($storecfg, $vmid, $device);
3532
3533         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3534         eval { qemu_deviceadd($vmid, $devicefull); };
3535         if (my $err = $@) {
3536             eval { qemu_drivedel($vmid, $deviceid); };
3537             warn $@ if $@;
3538             die $err;
3539         }
3540
3541     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3542
3543         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3544
3545         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3546         my $use_old_bios_files = undef;
3547         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3548
3549         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3550         qemu_deviceadd($vmid, $netdevicefull);
3551         eval { qemu_deviceaddverify($vmid, $deviceid); };
3552         if (my $err = $@) {
3553             eval { qemu_netdevdel($vmid, $deviceid); };
3554             warn $@ if $@;
3555             die $err;
3556         }
3557
3558     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3559
3560         my $bridgeid = $2;
3561         my $pciaddr = print_pci_addr($deviceid);
3562         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3563
3564         qemu_deviceadd($vmid, $devicefull);
3565         qemu_deviceaddverify($vmid, $deviceid);
3566
3567     } else {
3568         die "can't hotplug device '$deviceid'\n";
3569     }
3570
3571     return 1;
3572 }
3573
3574 # fixme: this should raise exceptions on error!
3575 sub vm_deviceunplug {
3576     my ($vmid, $conf, $deviceid) = @_;
3577
3578     my $devices_list = vm_devices_list($vmid);
3579     return 1 if !defined($devices_list->{$deviceid});
3580
3581     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3582
3583     if ($deviceid eq 'tablet') {
3584
3585         qemu_devicedel($vmid, $deviceid);
3586
3587     } elsif ($deviceid =~ m/^usb\d+$/) {
3588
3589         die "usb hotplug currently not reliable\n";
3590         # when unplugging usb devices this way,
3591         # there may be remaining usb controllers/hubs
3592         # so we disable it for now
3593         qemu_devicedel($vmid, $deviceid);
3594         qemu_devicedelverify($vmid, $deviceid);
3595
3596     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3597
3598         qemu_devicedel($vmid, $deviceid);
3599         qemu_devicedelverify($vmid, $deviceid);
3600         qemu_drivedel($vmid, $deviceid);
3601         qemu_iothread_del($conf, $vmid, $deviceid);
3602
3603     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3604
3605         qemu_devicedel($vmid, $deviceid);
3606         qemu_devicedelverify($vmid, $deviceid);
3607         qemu_iothread_del($conf, $vmid, $deviceid);
3608
3609     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3610
3611         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3612         my $device = parse_drive($deviceid, $conf->{$deviceid});
3613         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3614
3615         qemu_devicedel($vmid, $deviceid);
3616         qemu_drivedel($vmid, $deviceid);
3617         qemu_deletescsihw($conf, $vmid, $deviceid);
3618
3619     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3620
3621         qemu_devicedel($vmid, $deviceid);
3622         qemu_devicedelverify($vmid, $deviceid);
3623         qemu_netdevdel($vmid, $deviceid);
3624
3625     } else {
3626         die "can't unplug device '$deviceid'\n";
3627     }
3628
3629     return 1;
3630 }
3631
3632 sub qemu_deviceadd {
3633     my ($vmid, $devicefull) = @_;
3634
3635     $devicefull = "driver=".$devicefull;
3636     my %options =  split(/[=,]/, $devicefull);
3637
3638     vm_mon_cmd($vmid, "device_add" , %options);
3639 }
3640
3641 sub qemu_devicedel {
3642     my ($vmid, $deviceid) = @_;
3643
3644     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3645 }
3646
3647 sub qemu_iothread_add {
3648     my($vmid, $deviceid, $device) = @_;
3649
3650     if ($device->{iothread}) {
3651         my $iothreads = vm_iothreads_list($vmid);
3652         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3653     }
3654 }
3655
3656 sub qemu_iothread_del {
3657     my($conf, $vmid, $deviceid) = @_;
3658
3659     my $device = parse_drive($deviceid, $conf->{$deviceid});
3660     if ($device->{iothread}) {
3661         my $iothreads = vm_iothreads_list($vmid);
3662         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3663     }
3664 }
3665
3666 sub qemu_objectadd {
3667     my($vmid, $objectid, $qomtype) = @_;
3668
3669     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3670
3671     return 1;
3672 }
3673
3674 sub qemu_objectdel {
3675     my($vmid, $objectid) = @_;
3676
3677     vm_mon_cmd($vmid, "object-del", id => $objectid);
3678
3679     return 1;
3680 }
3681
3682 sub qemu_driveadd {
3683     my ($storecfg, $vmid, $device) = @_;
3684
3685     my $drive = print_drive_full($storecfg, $vmid, $device);
3686     $drive =~ s/\\/\\\\/g;
3687     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3688
3689     # If the command succeeds qemu prints: "OK"
3690     return 1 if $ret =~ m/OK/s;
3691
3692     die "adding drive failed: $ret\n";
3693 }
3694
3695 sub qemu_drivedel {
3696     my($vmid, $deviceid) = @_;
3697
3698     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3699     $ret =~ s/^\s+//;
3700
3701     return 1 if $ret eq "";
3702
3703     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3704     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3705
3706     die "deleting drive $deviceid failed : $ret\n";
3707 }
3708
3709 sub qemu_deviceaddverify {
3710     my ($vmid, $deviceid) = @_;
3711
3712     for (my $i = 0; $i <= 5; $i++) {
3713          my $devices_list = vm_devices_list($vmid);
3714          return 1 if defined($devices_list->{$deviceid});
3715          sleep 1;
3716     }
3717
3718     die "error on hotplug device '$deviceid'\n";
3719 }
3720
3721
3722 sub qemu_devicedelverify {
3723     my ($vmid, $deviceid) = @_;
3724
3725     # need to verify that the device is correctly removed as device_del
3726     # is async and empty return is not reliable
3727
3728     for (my $i = 0; $i <= 5; $i++) {
3729          my $devices_list = vm_devices_list($vmid);
3730          return 1 if !defined($devices_list->{$deviceid});
3731          sleep 1;
3732     }
3733
3734     die "error on hot-unplugging device '$deviceid'\n";
3735 }
3736
3737 sub qemu_findorcreatescsihw {
3738     my ($storecfg, $conf, $vmid, $device) = @_;
3739
3740     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3741
3742     my $scsihwid="$controller_prefix$controller";
3743     my $devices_list = vm_devices_list($vmid);
3744
3745     if(!defined($devices_list->{$scsihwid})) {
3746         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3747     }
3748
3749     return 1;
3750 }
3751
3752 sub qemu_deletescsihw {
3753     my ($conf, $vmid, $opt) = @_;
3754
3755     my $device = parse_drive($opt, $conf->{$opt});
3756
3757     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3758         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3759         return 1;
3760     }
3761
3762     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3763
3764     my $devices_list = vm_devices_list($vmid);
3765     foreach my $opt (keys %{$devices_list}) {
3766         if (PVE::QemuServer::is_valid_drivename($opt)) {
3767             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3768             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3769                 return 1;
3770             }
3771         }
3772     }
3773
3774     my $scsihwid="scsihw$controller";
3775
3776     vm_deviceunplug($vmid, $conf, $scsihwid);
3777
3778     return 1;
3779 }
3780
3781 sub qemu_add_pci_bridge {
3782     my ($storecfg, $conf, $vmid, $device) = @_;
3783
3784     my $bridges = {};
3785
3786     my $bridgeid;
3787
3788     print_pci_addr($device, $bridges);
3789
3790     while (my ($k, $v) = each %$bridges) {
3791         $bridgeid = $k;
3792     }
3793     return 1 if !defined($bridgeid) || $bridgeid < 1;
3794
3795     my $bridge = "pci.$bridgeid";
3796     my $devices_list = vm_devices_list($vmid);
3797
3798     if (!defined($devices_list->{$bridge})) {
3799         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3800     }
3801
3802     return 1;
3803 }
3804
3805 sub qemu_set_link_status {
3806     my ($vmid, $device, $up) = @_;
3807
3808     vm_mon_cmd($vmid, "set_link", name => $device,
3809                up => $up ? JSON::true : JSON::false);
3810 }
3811
3812 sub qemu_netdevadd {
3813     my ($vmid, $conf, $device, $deviceid) = @_;
3814
3815     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3816     my %options =  split(/[=,]/, $netdev);
3817
3818     vm_mon_cmd($vmid, "netdev_add",  %options);
3819     return 1;
3820 }
3821
3822 sub qemu_netdevdel {
3823     my ($vmid, $deviceid) = @_;
3824
3825     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3826 }
3827
3828 sub qemu_usb_hotplug {
3829     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3830
3831     return if !$device;
3832
3833     # remove the old one first
3834     vm_deviceunplug($vmid, $conf, $deviceid);
3835
3836     # check if xhci controller is necessary and available
3837     if ($device->{usb3}) {
3838
3839         my $devicelist = vm_devices_list($vmid);
3840
3841         if (!$devicelist->{xhci}) {
3842             my $pciaddr = print_pci_addr("xhci");
3843             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3844         }
3845     }
3846     my $d = parse_usb_device($device->{host});
3847     $d->{usb3} = $device->{usb3};
3848
3849     # add the new one
3850     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3851 }
3852
3853 sub qemu_cpu_hotplug {
3854     my ($vmid, $conf, $vcpus) = @_;
3855
3856     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3857
3858     my $sockets = 1;
3859     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3860     $sockets = $conf->{sockets} if  $conf->{sockets};
3861     my $cores = $conf->{cores} || 1;
3862     my $maxcpus = $sockets * $cores;
3863
3864     $vcpus = $maxcpus if !$vcpus;
3865
3866     die "you can't add more vcpus than maxcpus\n"
3867         if $vcpus > $maxcpus;
3868
3869     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3870
3871     if ($vcpus < $currentvcpus) {
3872
3873         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3874
3875             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3876                 qemu_devicedel($vmid, "cpu$i");
3877                 my $retry = 0;
3878                 my $currentrunningvcpus = undef;
3879                 while (1) {
3880                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3881                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3882                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3883                     $retry++;
3884                     sleep 1;
3885                 }
3886                 #update conf after each succesfull cpu unplug
3887                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3888                 PVE::QemuConfig->write_config($vmid, $conf);
3889             }
3890         } else {
3891             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3892         }
3893
3894         return;
3895     }
3896
3897     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3898     die "vcpus in running vm does not match its configuration\n"
3899         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3900
3901     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3902
3903         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3904             my $cpustr = print_cpu_device($conf, $i);
3905             qemu_deviceadd($vmid, $cpustr);
3906
3907             my $retry = 0;
3908             my $currentrunningvcpus = undef;
3909             while (1) {
3910                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3911                 last if scalar(@{$currentrunningvcpus}) == $i;
3912                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3913                 sleep 1;
3914                 $retry++;
3915             }
3916             #update conf after each succesfull cpu hotplug
3917             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3918             PVE::QemuConfig->write_config($vmid, $conf);
3919         }
3920     } else {
3921
3922         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3923             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3924         }
3925     }
3926 }
3927
3928 sub qemu_block_set_io_throttle {
3929     my ($vmid, $deviceid,
3930         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3931         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3932         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3933         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3934
3935     return if !check_running($vmid) ;
3936
3937     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3938         bps => int($bps),
3939         bps_rd => int($bps_rd),
3940         bps_wr => int($bps_wr),
3941         iops => int($iops),
3942         iops_rd => int($iops_rd),
3943         iops_wr => int($iops_wr),
3944         bps_max => int($bps_max),
3945         bps_rd_max => int($bps_rd_max),
3946         bps_wr_max => int($bps_wr_max),
3947         iops_max => int($iops_max),
3948         iops_rd_max => int($iops_rd_max),
3949         iops_wr_max => int($iops_wr_max),
3950         bps_max_length => int($bps_max_length),
3951         bps_rd_max_length => int($bps_rd_max_length),
3952         bps_wr_max_length => int($bps_wr_max_length),
3953         iops_max_length => int($iops_max_length),
3954         iops_rd_max_length => int($iops_rd_max_length),
3955         iops_wr_max_length => int($iops_wr_max_length),
3956     );
3957
3958 }
3959
3960 # old code, only used to shutdown old VM after update
3961 sub __read_avail {
3962     my ($fh, $timeout) = @_;
3963
3964     my $sel = new IO::Select;
3965     $sel->add($fh);
3966
3967     my $res = '';
3968     my $buf;
3969
3970     my @ready;
3971     while (scalar (@ready = $sel->can_read($timeout))) {
3972         my $count;
3973         if ($count = $fh->sysread($buf, 8192)) {
3974             if ($buf =~ /^(.*)\(qemu\) $/s) {
3975                 $res .= $1;
3976                 last;
3977             } else {
3978                 $res .= $buf;
3979             }
3980         } else {
3981             if (!defined($count)) {
3982                 die "$!\n";
3983             }
3984             last;
3985         }
3986     }
3987
3988     die "monitor read timeout\n" if !scalar(@ready);
3989
3990     return $res;
3991 }
3992
3993 # old code, only used to shutdown old VM after update
3994 sub vm_monitor_command {
3995     my ($vmid, $cmdstr, $nocheck) = @_;
3996
3997     my $res;
3998
3999     eval {
4000         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4001
4002         my $sname = "${var_run_tmpdir}/$vmid.mon";
4003
4004         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
4005             die "unable to connect to VM $vmid socket - $!\n";
4006
4007         my $timeout = 3;
4008
4009         # hack: migrate sometime blocks the monitor (when migrate_downtime
4010         # is set)
4011         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4012             $timeout = 60*60; # 1 hour
4013         }
4014
4015         # read banner;
4016         my $data = __read_avail($sock, $timeout);
4017
4018         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
4019             die "got unexpected qemu monitor banner\n";
4020         }
4021
4022         my $sel = new IO::Select;
4023         $sel->add($sock);
4024
4025         if (!scalar(my @ready = $sel->can_write($timeout))) {
4026             die "monitor write error - timeout";
4027         }
4028
4029         my $fullcmd = "$cmdstr\r";
4030
4031         # syslog('info', "VM $vmid monitor command: $cmdstr");
4032
4033         my $b;
4034         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
4035             die "monitor write error - $!";
4036         }
4037
4038         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4039
4040         $timeout = 20;
4041
4042         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4043             $timeout = 60*60; # 1 hour
4044         } elsif ($cmdstr =~ m/^(eject|change)/) {
4045             $timeout = 60; # note: cdrom mount command is slow
4046         }
4047         if ($res = __read_avail($sock, $timeout)) {
4048
4049             my @lines = split("\r?\n", $res);
4050
4051             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4052
4053             $res = join("\n", @lines);
4054             $res .= "\n";
4055         }
4056     };
4057
4058     my $err = $@;
4059
4060     if ($err) {
4061         syslog("err", "VM $vmid monitor command failed - $err");
4062         die $err;
4063     }
4064
4065     return $res;
4066 }
4067
4068 sub qemu_block_resize {
4069     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4070
4071     my $running = check_running($vmid);
4072
4073     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4074
4075     return if !$running;
4076
4077     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4078
4079 }
4080
4081 sub qemu_volume_snapshot {
4082     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4083
4084     my $running = check_running($vmid);
4085
4086     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4087         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4088     } else {
4089         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4090     }
4091 }
4092
4093 sub qemu_volume_snapshot_delete {
4094     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4095
4096     my $running = check_running($vmid);
4097
4098     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4099         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4100     } else {
4101         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4102     }
4103 }
4104
4105 sub set_migration_caps {
4106     my ($vmid) = @_;
4107
4108     my $cap_ref = [];
4109
4110     my $enabled_cap = {
4111         "auto-converge" => 1,
4112         "xbzrle" => 1,
4113         "x-rdma-pin-all" => 0,
4114         "zero-blocks" => 0,
4115         "compress" => 0
4116     };
4117
4118     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4119
4120     for my $supported_capability (@$supported_capabilities) {
4121         push @$cap_ref, {
4122             capability => $supported_capability->{capability},
4123             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4124         };
4125     }
4126
4127     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4128 }
4129
4130 my $fast_plug_option = {
4131     'lock' => 1,
4132     'name' => 1,
4133     'onboot' => 1,
4134     'shares' => 1,
4135     'startup' => 1,
4136     'description' => 1,
4137     'protection' => 1,
4138     'vmstatestorage' => 1,
4139 };
4140
4141 # hotplug changes in [PENDING]
4142 # $selection hash can be used to only apply specified options, for
4143 # example: { cores => 1 } (only apply changed 'cores')
4144 # $errors ref is used to return error messages
4145 sub vmconfig_hotplug_pending {
4146     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4147
4148     my $defaults = load_defaults();
4149
4150     # commit values which do not have any impact on running VM first
4151     # Note: those option cannot raise errors, we we do not care about
4152     # $selection and always apply them.
4153
4154     my $add_error = sub {
4155         my ($opt, $msg) = @_;
4156         $errors->{$opt} = "hotplug problem - $msg";
4157     };
4158
4159     my $changes = 0;
4160     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4161         if ($fast_plug_option->{$opt}) {
4162             $conf->{$opt} = $conf->{pending}->{$opt};
4163             delete $conf->{pending}->{$opt};
4164             $changes = 1;
4165         }
4166     }
4167
4168     if ($changes) {
4169         PVE::QemuConfig->write_config($vmid, $conf);
4170         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4171     }
4172
4173     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4174
4175     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4176     while (my ($opt, $force) = each %$pending_delete_hash) {
4177         next if $selection && !$selection->{$opt};
4178         eval {
4179             if ($opt eq 'hotplug') {
4180                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4181             } elsif ($opt eq 'tablet') {
4182                 die "skip\n" if !$hotplug_features->{usb};
4183                 if ($defaults->{tablet}) {
4184                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4185                 } else {
4186                     vm_deviceunplug($vmid, $conf, $opt);
4187                 }
4188             } elsif ($opt =~ m/^usb\d+/) {
4189                 die "skip\n";
4190                 # since we cannot reliably hot unplug usb devices
4191                 # we are disabling it
4192                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4193                 vm_deviceunplug($vmid, $conf, $opt);
4194             } elsif ($opt eq 'vcpus') {
4195                 die "skip\n" if !$hotplug_features->{cpu};
4196                 qemu_cpu_hotplug($vmid, $conf, undef);
4197             } elsif ($opt eq 'balloon') {
4198                 # enable balloon device is not hotpluggable
4199                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4200             } elsif ($fast_plug_option->{$opt}) {
4201                 # do nothing
4202             } elsif ($opt =~ m/^net(\d+)$/) {
4203                 die "skip\n" if !$hotplug_features->{network};
4204                 vm_deviceunplug($vmid, $conf, $opt);
4205             } elsif (is_valid_drivename($opt)) {
4206                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4207                 vm_deviceunplug($vmid, $conf, $opt);
4208                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4209             } elsif ($opt =~ m/^memory$/) {
4210                 die "skip\n" if !$hotplug_features->{memory};
4211                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4212             } elsif ($opt eq 'cpuunits') {
4213                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4214             } elsif ($opt eq 'cpulimit') {
4215                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4216             } else {
4217                 die "skip\n";
4218             }
4219         };
4220         if (my $err = $@) {
4221             &$add_error($opt, $err) if $err ne "skip\n";
4222         } else {
4223             # save new config if hotplug was successful
4224             delete $conf->{$opt};
4225             vmconfig_undelete_pending_option($conf, $opt);
4226             PVE::QemuConfig->write_config($vmid, $conf);
4227             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4228         }
4229     }
4230
4231     foreach my $opt (keys %{$conf->{pending}}) {
4232         next if $selection && !$selection->{$opt};
4233         my $value = $conf->{pending}->{$opt};
4234         eval {
4235             if ($opt eq 'hotplug') {
4236                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4237             } elsif ($opt eq 'tablet') {
4238                 die "skip\n" if !$hotplug_features->{usb};
4239                 if ($value == 1) {
4240                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4241                 } elsif ($value == 0) {
4242                     vm_deviceunplug($vmid, $conf, $opt);
4243                 }
4244             } elsif ($opt =~ m/^usb\d+$/) {
4245                 die "skip\n";
4246                 # since we cannot reliably hot unplug usb devices
4247                 # we are disabling it
4248                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4249                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4250                 die "skip\n" if !$d;
4251                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4252             } elsif ($opt eq 'vcpus') {
4253                 die "skip\n" if !$hotplug_features->{cpu};
4254                 qemu_cpu_hotplug($vmid, $conf, $value);
4255             } elsif ($opt eq 'balloon') {
4256                 # enable/disable balloning device is not hotpluggable
4257                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4258                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4259                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4260
4261                 # allow manual ballooning if shares is set to zero
4262                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4263                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4264                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4265                 }
4266             } elsif ($opt =~ m/^net(\d+)$/) {
4267                 # some changes can be done without hotplug
4268                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4269                                     $vmid, $opt, $value);
4270             } elsif (is_valid_drivename($opt)) {
4271                 # some changes can be done without hotplug
4272                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4273                                      $vmid, $opt, $value, 1);
4274             } elsif ($opt =~ m/^memory$/) { #dimms
4275                 die "skip\n" if !$hotplug_features->{memory};
4276                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4277             } elsif ($opt eq 'cpuunits') {
4278                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4279             } elsif ($opt eq 'cpulimit') {
4280                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4281                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4282             } else {
4283                 die "skip\n";  # skip non-hot-pluggable options
4284             }
4285         };
4286         if (my $err = $@) {
4287             &$add_error($opt, $err) if $err ne "skip\n";
4288         } else {
4289             # save new config if hotplug was successful
4290             $conf->{$opt} = $value;
4291             delete $conf->{pending}->{$opt};
4292             PVE::QemuConfig->write_config($vmid, $conf);
4293             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4294         }
4295     }
4296 }
4297
4298 sub try_deallocate_drive {
4299     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4300
4301     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4302         my $volid = $drive->{file};
4303         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4304             my $sid = PVE::Storage::parse_volume_id($volid);
4305             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4306
4307             # check if the disk is really unused
4308             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4309                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4310             PVE::Storage::vdisk_free($storecfg, $volid);
4311             return 1;
4312         } else {
4313             # If vm is not owner of this disk remove from config
4314             return 1;
4315         }
4316     }
4317
4318     return undef;
4319 }
4320
4321 sub vmconfig_delete_or_detach_drive {
4322     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4323
4324     my $drive = parse_drive($opt, $conf->{$opt});
4325
4326     my $rpcenv = PVE::RPCEnvironment::get();
4327     my $authuser = $rpcenv->get_user();
4328
4329     if ($force) {
4330         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4331         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4332     } else {
4333         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4334     }
4335 }
4336
4337 sub vmconfig_apply_pending {
4338     my ($vmid, $conf, $storecfg) = @_;
4339
4340     # cold plug
4341
4342     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4343     while (my ($opt, $force) = each %$pending_delete_hash) {
4344         die "internal error" if $opt =~ m/^unused/;
4345         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4346         if (!defined($conf->{$opt})) {
4347             vmconfig_undelete_pending_option($conf, $opt);
4348             PVE::QemuConfig->write_config($vmid, $conf);
4349         } elsif (is_valid_drivename($opt)) {
4350             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4351             vmconfig_undelete_pending_option($conf, $opt);
4352             delete $conf->{$opt};
4353             PVE::QemuConfig->write_config($vmid, $conf);
4354         } else {
4355             vmconfig_undelete_pending_option($conf, $opt);
4356             delete $conf->{$opt};
4357             PVE::QemuConfig->write_config($vmid, $conf);
4358         }
4359     }
4360
4361     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4362
4363     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4364         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4365
4366         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4367             # skip if nothing changed
4368         } elsif (is_valid_drivename($opt)) {
4369             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4370                 if defined($conf->{$opt});
4371             $conf->{$opt} = $conf->{pending}->{$opt};
4372         } else {
4373             $conf->{$opt} = $conf->{pending}->{$opt};
4374         }
4375
4376         delete $conf->{pending}->{$opt};
4377         PVE::QemuConfig->write_config($vmid, $conf);
4378     }
4379 }
4380
4381 my $safe_num_ne = sub {
4382     my ($a, $b) = @_;
4383
4384     return 0 if !defined($a) && !defined($b);
4385     return 1 if !defined($a);
4386     return 1 if !defined($b);
4387
4388     return $a != $b;
4389 };
4390
4391 my $safe_string_ne = sub {
4392     my ($a, $b) = @_;
4393
4394     return 0 if !defined($a) && !defined($b);
4395     return 1 if !defined($a);
4396     return 1 if !defined($b);
4397
4398     return $a ne $b;
4399 };
4400
4401 sub vmconfig_update_net {
4402     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4403
4404     my $newnet = parse_net($value);
4405
4406     if ($conf->{$opt}) {
4407         my $oldnet = parse_net($conf->{$opt});
4408
4409         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4410             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4411             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4412             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4413
4414             # for non online change, we try to hot-unplug
4415             die "skip\n" if !$hotplug;
4416             vm_deviceunplug($vmid, $conf, $opt);
4417         } else {
4418
4419             die "internal error" if $opt !~ m/net(\d+)/;
4420             my $iface = "tap${vmid}i$1";
4421
4422             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4423                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4424                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4425                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4426                 PVE::Network::tap_unplug($iface);
4427                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4428             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4429                 # Rate can be applied on its own but any change above needs to
4430                 # include the rate in tap_plug since OVS resets everything.
4431                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4432             }
4433
4434             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4435                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4436             }
4437
4438             return 1;
4439         }
4440     }
4441
4442     if ($hotplug) {
4443         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4444     } else {
4445         die "skip\n";
4446     }
4447 }
4448
4449 sub vmconfig_update_disk {
4450     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4451
4452     # fixme: do we need force?
4453
4454     my $drive = parse_drive($opt, $value);
4455
4456     if ($conf->{$opt}) {
4457
4458         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4459
4460             my $media = $drive->{media} || 'disk';
4461             my $oldmedia = $old_drive->{media} || 'disk';
4462             die "unable to change media type\n" if $media ne $oldmedia;
4463
4464             if (!drive_is_cdrom($old_drive)) {
4465
4466                 if ($drive->{file} ne $old_drive->{file}) {
4467
4468                     die "skip\n" if !$hotplug;
4469
4470                     # unplug and register as unused
4471                     vm_deviceunplug($vmid, $conf, $opt);
4472                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4473
4474                 } else {
4475                     # update existing disk
4476
4477                     # skip non hotpluggable value
4478                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4479                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4480                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4481                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4482                         die "skip\n";
4483                     }
4484
4485                     # apply throttle
4486                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4487                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4488                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4489                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4490                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4491                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4492                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4493                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4494                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4495                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4496                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4497                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4498                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4499                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4500                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4501                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4502                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4503                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4504
4505                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4506                                                    ($drive->{mbps} || 0)*1024*1024,
4507                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4508                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4509                                                    $drive->{iops} || 0,
4510                                                    $drive->{iops_rd} || 0,
4511                                                    $drive->{iops_wr} || 0,
4512                                                    ($drive->{mbps_max} || 0)*1024*1024,
4513                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4514                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4515                                                    $drive->{iops_max} || 0,
4516                                                    $drive->{iops_rd_max} || 0,
4517                                                    $drive->{iops_wr_max} || 0,
4518                                                    $drive->{bps_max_length} || 1,
4519                                                    $drive->{bps_rd_max_length} || 1,
4520                                                    $drive->{bps_wr_max_length} || 1,
4521                                                    $drive->{iops_max_length} || 1,
4522                                                    $drive->{iops_rd_max_length} || 1,
4523                                                    $drive->{iops_wr_max_length} || 1);
4524
4525                     }
4526
4527                     return 1;
4528                 }
4529
4530             } else { # cdrom
4531
4532                 if ($drive->{file} eq 'none') {
4533                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4534                 } else {
4535                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4536                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4537                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4538                 }
4539
4540                 return 1;
4541             }
4542         }
4543     }
4544
4545     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4546     # hotplug new disks
4547     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4548     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4549 }
4550
4551 sub vm_start {
4552     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4553         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4554
4555     PVE::QemuConfig->lock_config($vmid, sub {
4556         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4557
4558         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4559
4560         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4561
4562         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4563
4564         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4565             vmconfig_apply_pending($vmid, $conf, $storecfg);
4566             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4567         }
4568
4569         my $defaults = load_defaults();
4570
4571         # set environment variable useful inside network script
4572         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4573
4574         my $local_volumes = {};
4575
4576         if ($targetstorage) {
4577             foreach_drive($conf, sub {
4578                 my ($ds, $drive) = @_;
4579
4580                 return if drive_is_cdrom($drive);
4581
4582                 my $volid = $drive->{file};
4583
4584                 return if !$volid;
4585
4586                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4587
4588                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4589                 return if $scfg->{shared};
4590                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4591             });
4592
4593             my $format = undef;
4594
4595             foreach my $opt (sort keys %$local_volumes) {
4596
4597                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4598                 my $drive = parse_drive($opt, $conf->{$opt});
4599
4600                 #if remote storage is specified, use default format
4601                 if ($targetstorage && $targetstorage ne "1") {
4602                     $storeid = $targetstorage;
4603                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4604                     $format = $defFormat;
4605                 } else {
4606                     #else we use same format than original
4607                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4608                     $format = qemu_img_format($scfg, $volid);
4609                 }
4610
4611                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4612                 my $newdrive = $drive;
4613                 $newdrive->{format} = $format;
4614                 $newdrive->{file} = $newvolid;
4615                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4616                 $local_volumes->{$opt} = $drivestr;
4617                 #pass drive to conf for command line
4618                 $conf->{$opt} = $drivestr;
4619             }
4620         }
4621
4622         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4623
4624         my $migrate_port = 0;
4625         my $migrate_uri;
4626         if ($statefile) {
4627             if ($statefile eq 'tcp') {
4628                 my $localip = "localhost";
4629                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4630                 my $nodename = PVE::INotify::nodename();
4631
4632                 if (!defined($migration_type)) {
4633                     if (defined($datacenterconf->{migration}->{type})) {
4634                         $migration_type = $datacenterconf->{migration}->{type};
4635                     } else {
4636                         $migration_type = 'secure';
4637                     }
4638                 }
4639
4640                 if ($migration_type eq 'insecure') {
4641                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4642                     if ($migrate_network_addr) {
4643                         $localip = $migrate_network_addr;
4644                     } else {
4645                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4646                     }
4647
4648                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4649                 }
4650
4651                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4652                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4653                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4654                 push @$cmd, '-incoming', $migrate_uri;
4655                 push @$cmd, '-S';
4656
4657             } elsif ($statefile eq 'unix') {
4658                 # should be default for secure migrations as a ssh TCP forward
4659                 # tunnel is not deterministic reliable ready and fails regurarly
4660                 # to set up in time, so use UNIX socket forwards
4661                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4662                 unlink $socket_addr;
4663
4664                 $migrate_uri = "unix:$socket_addr";
4665
4666                 push @$cmd, '-incoming', $migrate_uri;
4667                 push @$cmd, '-S';
4668
4669             } else {
4670                 push @$cmd, '-loadstate', $statefile;
4671             }
4672         } elsif ($paused) {
4673             push @$cmd, '-S';
4674         }
4675
4676         # host pci devices
4677         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4678           my $d = parse_hostpci($conf->{"hostpci$i"});
4679           next if !$d;
4680           my $pcidevices = $d->{pciid};
4681           foreach my $pcidevice (@$pcidevices) {
4682                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4683
4684                 my $info = pci_device_info("0000:$pciid");
4685                 die "IOMMU not present\n" if !check_iommu_support();
4686                 die "no pci device info for device '$pciid'\n" if !$info;
4687                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4688                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4689           }
4690         }
4691
4692         PVE::Storage::activate_volumes($storecfg, $vollist);
4693
4694         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4695             my $cmd = [];
4696             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4697             eval  { run_command($cmd); };
4698         }
4699
4700         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4701                                                   : $defaults->{cpuunits};
4702
4703         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4704         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4705
4706         my %properties = (
4707             Slice => 'qemu.slice',
4708             KillMode => 'none',
4709             CPUShares => $cpuunits
4710         );
4711
4712         if (my $cpulimit = $conf->{cpulimit}) {
4713             $properties{CPUQuota} = int($cpulimit * 100);
4714         }
4715         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4716
4717         if ($conf->{hugepages}) {
4718
4719             my $code = sub {
4720                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4721                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4722
4723                 PVE::QemuServer::Memory::hugepages_mount();
4724                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4725
4726                 eval  {
4727                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4728                     run_command($cmd, %run_params);
4729                 };
4730
4731                 if (my $err = $@) {
4732                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4733                     die $err;
4734                 }
4735
4736                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4737             };
4738             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4739
4740         } else {
4741             eval  {
4742                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4743                 run_command($cmd, %run_params);
4744             };
4745         }
4746
4747         if (my $err = $@) {
4748             # deactivate volumes if start fails
4749             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4750             die "start failed: $err";
4751         }
4752
4753         print "migration listens on $migrate_uri\n" if $migrate_uri;
4754
4755         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4756             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4757             warn $@ if $@;
4758         }
4759
4760         #start nbd server for storage migration
4761         if ($targetstorage) {
4762             my $nodename = PVE::INotify::nodename();
4763             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4764             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4765             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4766             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4767
4768             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4769
4770             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4771
4772             foreach my $opt (sort keys %$local_volumes) {
4773                 my $volid = $local_volumes->{$opt};
4774                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4775                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4776                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4777             }
4778         }
4779
4780         if ($migratedfrom) {
4781             eval {
4782                 set_migration_caps($vmid);
4783             };
4784             warn $@ if $@;
4785
4786             if ($spice_port) {
4787                 print "spice listens on port $spice_port\n";
4788                 if ($spice_ticket) {
4789                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4790                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4791                 }
4792             }
4793
4794         } else {
4795             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4796                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4797                     if $conf->{balloon};
4798             }
4799
4800             foreach my $opt (keys %$conf) {
4801                 next if $opt !~  m/^net\d+$/;
4802                 my $nicconf = parse_net($conf->{$opt});
4803                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4804             }
4805         }
4806
4807         vm_mon_cmd_nocheck($vmid, 'qom-set',
4808                     path => "machine/peripheral/balloon0",
4809                     property => "guest-stats-polling-interval",
4810                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4811
4812     });
4813 }
4814
4815 sub vm_mon_cmd {
4816     my ($vmid, $execute, %params) = @_;
4817
4818     my $cmd = { execute => $execute, arguments => \%params };
4819     vm_qmp_command($vmid, $cmd);
4820 }
4821
4822 sub vm_mon_cmd_nocheck {
4823     my ($vmid, $execute, %params) = @_;
4824
4825     my $cmd = { execute => $execute, arguments => \%params };
4826     vm_qmp_command($vmid, $cmd, 1);
4827 }
4828
4829 sub vm_qmp_command {
4830     my ($vmid, $cmd, $nocheck) = @_;
4831
4832     my $res;
4833
4834     my $timeout;
4835     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4836         $timeout = $cmd->{arguments}->{timeout};
4837         delete $cmd->{arguments}->{timeout};
4838     }
4839
4840     eval {
4841         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4842         my $sname = qmp_socket($vmid);
4843         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4844             my $qmpclient = PVE::QMPClient->new();
4845
4846             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4847         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4848             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4849                 if scalar(%{$cmd->{arguments}});
4850             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4851         } else {
4852             die "unable to open monitor socket\n";
4853         }
4854     };
4855     if (my $err = $@) {
4856         syslog("err", "VM $vmid qmp command failed - $err");
4857         die $err;
4858     }
4859
4860     return $res;
4861 }
4862
4863 sub vm_human_monitor_command {
4864     my ($vmid, $cmdline) = @_;
4865
4866     my $res;
4867
4868     my $cmd = {
4869         execute => 'human-monitor-command',
4870         arguments => { 'command-line' => $cmdline},
4871     };
4872
4873     return vm_qmp_command($vmid, $cmd);
4874 }
4875
4876 sub vm_commandline {
4877     my ($storecfg, $vmid) = @_;
4878
4879     my $conf = PVE::QemuConfig->load_config($vmid);
4880
4881     my $defaults = load_defaults();
4882
4883     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4884
4885     return PVE::Tools::cmd2string($cmd);
4886 }
4887
4888 sub vm_reset {
4889     my ($vmid, $skiplock) = @_;
4890
4891     PVE::QemuConfig->lock_config($vmid, sub {
4892
4893         my $conf = PVE::QemuConfig->load_config($vmid);
4894
4895         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4896
4897         vm_mon_cmd($vmid, "system_reset");
4898     });
4899 }
4900
4901 sub get_vm_volumes {
4902     my ($conf) = @_;
4903
4904     my $vollist = [];
4905     foreach_volid($conf, sub {
4906         my ($volid, $attr) = @_;
4907
4908         return if $volid =~ m|^/|;
4909
4910         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4911         return if !$sid;
4912
4913         push @$vollist, $volid;
4914     });
4915
4916     return $vollist;
4917 }
4918
4919 sub vm_stop_cleanup {
4920     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4921
4922     eval {
4923
4924         if (!$keepActive) {
4925             my $vollist = get_vm_volumes($conf);
4926             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4927         }
4928
4929         foreach my $ext (qw(mon qmp pid vnc qga)) {
4930             unlink "/var/run/qemu-server/${vmid}.$ext";
4931         }
4932
4933         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4934     };
4935     warn $@ if $@; # avoid errors - just warn
4936 }
4937
4938 # Note: use $nockeck to skip tests if VM configuration file exists.
4939 # We need that when migration VMs to other nodes (files already moved)
4940 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4941 sub vm_stop {
4942     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4943
4944     $force = 1 if !defined($force) && !$shutdown;
4945
4946     if ($migratedfrom){
4947         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4948         kill 15, $pid if $pid;
4949         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4950         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4951         return;
4952     }
4953
4954     PVE::QemuConfig->lock_config($vmid, sub {
4955
4956         my $pid = check_running($vmid, $nocheck);
4957         return if !$pid;
4958
4959         my $conf;
4960         if (!$nocheck) {
4961             $conf = PVE::QemuConfig->load_config($vmid);
4962             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4963             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4964                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4965                 $timeout = $opts->{down} if $opts->{down};
4966             }
4967         }
4968
4969         $timeout = 60 if !defined($timeout);
4970
4971         eval {
4972             if ($shutdown) {
4973                 if (defined($conf) && $conf->{agent}) {
4974                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4975                 } else {
4976                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4977                 }
4978             } else {
4979                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4980             }
4981         };
4982         my $err = $@;
4983
4984         if (!$err) {
4985             my $count = 0;
4986             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4987                 $count++;
4988                 sleep 1;
4989             }
4990
4991             if ($count >= $timeout) {
4992                 if ($force) {
4993                     warn "VM still running - terminating now with SIGTERM\n";
4994                     kill 15, $pid;
4995                 } else {
4996                     die "VM quit/powerdown failed - got timeout\n";
4997                 }
4998             } else {
4999                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5000                 return;
5001             }
5002         } else {
5003             if ($force) {
5004                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5005                 kill 15, $pid;
5006             } else {
5007                 die "VM quit/powerdown failed\n";
5008             }
5009         }
5010
5011         # wait again
5012         $timeout = 10;
5013
5014         my $count = 0;
5015         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5016             $count++;
5017             sleep 1;
5018         }
5019
5020         if ($count >= $timeout) {
5021             warn "VM still running - terminating now with SIGKILL\n";
5022             kill 9, $pid;
5023             sleep 1;
5024         }
5025
5026         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5027    });
5028 }
5029
5030 sub vm_suspend {
5031     my ($vmid, $skiplock) = @_;
5032
5033     PVE::QemuConfig->lock_config($vmid, sub {
5034
5035         my $conf = PVE::QemuConfig->load_config($vmid);
5036
5037         PVE::QemuConfig->check_lock($conf)
5038             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5039
5040         vm_mon_cmd($vmid, "stop");
5041     });
5042 }
5043
5044 sub vm_resume {
5045     my ($vmid, $skiplock, $nocheck) = @_;
5046
5047     PVE::QemuConfig->lock_config($vmid, sub {
5048
5049         if (!$nocheck) {
5050
5051             my $conf = PVE::QemuConfig->load_config($vmid);
5052
5053             PVE::QemuConfig->check_lock($conf)
5054                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5055
5056             vm_mon_cmd($vmid, "cont");
5057
5058         } else {
5059             vm_mon_cmd_nocheck($vmid, "cont");
5060         }
5061     });
5062 }
5063
5064 sub vm_sendkey {
5065     my ($vmid, $skiplock, $key) = @_;
5066
5067     PVE::QemuConfig->lock_config($vmid, sub {
5068
5069         my $conf = PVE::QemuConfig->load_config($vmid);
5070
5071         # there is no qmp command, so we use the human monitor command
5072         vm_human_monitor_command($vmid, "sendkey $key");
5073     });
5074 }
5075
5076 sub vm_destroy {
5077     my ($storecfg, $vmid, $skiplock) = @_;
5078
5079     PVE::QemuConfig->lock_config($vmid, sub {
5080
5081         my $conf = PVE::QemuConfig->load_config($vmid);
5082
5083         if (!check_running($vmid)) {
5084             destroy_vm($storecfg, $vmid, undef, $skiplock);
5085         } else {
5086             die "VM $vmid is running - destroy failed\n";
5087         }
5088     });
5089 }
5090
5091 # pci helpers
5092
5093 sub file_write {
5094     my ($filename, $buf) = @_;
5095
5096     my $fh = IO::File->new($filename, "w");
5097     return undef if !$fh;
5098
5099     my $res = print $fh $buf;
5100
5101     $fh->close();
5102
5103     return $res;
5104 }
5105
5106 sub pci_device_info {
5107     my ($name) = @_;
5108
5109     my $res;
5110
5111     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5112     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5113
5114     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5115     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5116
5117     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5118     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5119
5120     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5121     return undef if !defined($product) || $product !~ s/^0x//;
5122
5123     $res = {
5124         name => $name,
5125         vendor => $vendor,
5126         product => $product,
5127         domain => $domain,
5128         bus => $bus,
5129         slot => $slot,
5130         func => $func,
5131         irq => $irq,
5132         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5133     };
5134
5135     return $res;
5136 }
5137
5138 sub pci_dev_reset {
5139     my ($dev) = @_;
5140
5141     my $name = $dev->{name};
5142
5143     my $fn = "$pcisysfs/devices/$name/reset";
5144
5145     return file_write($fn, "1");
5146 }
5147
5148 sub pci_dev_bind_to_vfio {
5149     my ($dev) = @_;
5150
5151     my $name = $dev->{name};
5152
5153     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5154
5155     if (!-d $vfio_basedir) {
5156         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5157     }
5158     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5159
5160     my $testdir = "$vfio_basedir/$name";
5161     return 1 if -d $testdir;
5162
5163     my $data = "$dev->{vendor} $dev->{product}";
5164     return undef if !file_write("$vfio_basedir/new_id", $data);
5165
5166     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5167     if (!file_write($fn, $name)) {
5168         return undef if -f $fn;
5169     }
5170
5171     $fn = "$vfio_basedir/bind";
5172     if (! -d $testdir) {
5173         return undef if !file_write($fn, $name);
5174     }
5175
5176     return -d $testdir;
5177 }
5178
5179 sub pci_dev_group_bind_to_vfio {
5180     my ($pciid) = @_;
5181
5182     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5183
5184     if (!-d $vfio_basedir) {
5185         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5186     }
5187     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5188
5189     # get IOMMU group devices
5190     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5191       my @devs = grep /^0000:/, readdir($D);
5192     closedir($D);
5193
5194     foreach my $pciid (@devs) {
5195         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5196
5197         # pci bridges, switches or root ports are not supported
5198         # they have a pci_bus subdirectory so skip them
5199         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5200
5201         my $info = pci_device_info($1);
5202         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5203     }
5204
5205     return 1;
5206 }
5207
5208 # vzdump restore implementaion
5209
5210 sub tar_archive_read_firstfile {
5211     my $archive = shift;
5212
5213     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5214
5215     # try to detect archive type first
5216     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5217         die "unable to open file '$archive'\n";
5218     my $firstfile = <$fh>;
5219     kill 15, $pid;
5220     close $fh;
5221
5222     die "ERROR: archive contaions no data\n" if !$firstfile;
5223     chomp $firstfile;
5224
5225     return $firstfile;
5226 }
5227
5228 sub tar_restore_cleanup {
5229     my ($storecfg, $statfile) = @_;
5230
5231     print STDERR "starting cleanup\n";
5232
5233     if (my $fd = IO::File->new($statfile, "r")) {
5234         while (defined(my $line = <$fd>)) {
5235             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5236                 my $volid = $2;
5237                 eval {
5238                     if ($volid =~ m|^/|) {
5239                         unlink $volid || die 'unlink failed\n';
5240                     } else {
5241                         PVE::Storage::vdisk_free($storecfg, $volid);
5242                     }
5243                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5244                 };
5245                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5246             } else {
5247                 print STDERR "unable to parse line in statfile - $line";
5248             }
5249         }
5250         $fd->close();
5251     }
5252 }
5253
5254 sub restore_archive {
5255     my ($archive, $vmid, $user, $opts) = @_;
5256
5257     my $format = $opts->{format};
5258     my $comp;
5259
5260     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5261         $format = 'tar' if !$format;
5262         $comp = 'gzip';
5263     } elsif ($archive =~ m/\.tar$/) {
5264         $format = 'tar' if !$format;
5265     } elsif ($archive =~ m/.tar.lzo$/) {
5266         $format = 'tar' if !$format;
5267         $comp = 'lzop';
5268     } elsif ($archive =~ m/\.vma$/) {
5269         $format = 'vma' if !$format;
5270     } elsif ($archive =~ m/\.vma\.gz$/) {
5271         $format = 'vma' if !$format;
5272         $comp = 'gzip';
5273     } elsif ($archive =~ m/\.vma\.lzo$/) {
5274         $format = 'vma' if !$format;
5275         $comp = 'lzop';
5276     } else {
5277         $format = 'vma' if !$format; # default
5278     }
5279
5280     # try to detect archive format
5281     if ($format eq 'tar') {
5282         return restore_tar_archive($archive, $vmid, $user, $opts);
5283     } else {
5284         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5285     }
5286 }
5287
5288 sub restore_update_config_line {
5289     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5290
5291     return if $line =~ m/^\#qmdump\#/;
5292     return if $line =~ m/^\#vzdump\#/;
5293     return if $line =~ m/^lock:/;
5294     return if $line =~ m/^unused\d+:/;
5295     return if $line =~ m/^parent:/;
5296     return if $line =~ m/^template:/; # restored VM is never a template
5297
5298     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5299     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5300         # try to convert old 1.X settings
5301         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5302         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5303             my ($model, $macaddr) = split(/\=/, $devconfig);
5304             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5305             my $net = {
5306                 model => $model,
5307                 bridge => "vmbr$ind",
5308                 macaddr => $macaddr,
5309             };
5310             my $netstr = print_net($net);
5311
5312             print $outfd "net$cookie->{netcount}: $netstr\n";
5313             $cookie->{netcount}++;
5314         }
5315     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5316         my ($id, $netstr) = ($1, $2);
5317         my $net = parse_net($netstr);
5318         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5319         $netstr = print_net($net);
5320         print $outfd "$id: $netstr\n";
5321     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5322         my $virtdev = $1;
5323         my $value = $3;
5324         my $di = parse_drive($virtdev, $value);
5325         if (defined($di->{backup}) && !$di->{backup}) {
5326             print $outfd "#$line";
5327         } elsif ($map->{$virtdev}) {
5328             delete $di->{format}; # format can change on restore
5329             $di->{file} = $map->{$virtdev};
5330             $value = print_drive($vmid, $di);
5331             print $outfd "$virtdev: $value\n";
5332         } else {
5333             print $outfd $line;
5334         }
5335     } else {
5336         print $outfd $line;
5337     }
5338 }
5339
5340 sub scan_volids {
5341     my ($cfg, $vmid) = @_;
5342
5343     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5344
5345     my $volid_hash = {};
5346     foreach my $storeid (keys %$info) {
5347         foreach my $item (@{$info->{$storeid}}) {
5348             next if !($item->{volid} && $item->{size});
5349             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5350             $volid_hash->{$item->{volid}} = $item;
5351         }
5352     }
5353
5354     return $volid_hash;
5355 }
5356
5357 sub is_volume_in_use {
5358     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5359
5360     my $path = PVE::Storage::path($storecfg, $volid);
5361
5362     my $scan_config = sub {
5363         my ($cref, $snapname) = @_;
5364
5365         foreach my $key (keys %$cref) {
5366             my $value = $cref->{$key};
5367             if (is_valid_drivename($key)) {
5368                 next if $skip_drive && $key eq $skip_drive;
5369                 my $drive = parse_drive($key, $value);
5370                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5371                 return 1 if $volid eq $drive->{file};
5372                 if ($drive->{file} =~ m!^/!) {
5373                     return 1 if $drive->{file} eq $path;
5374                 } else {
5375                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5376                     next if !$storeid;
5377                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5378                     next if !$scfg;
5379                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5380                 }
5381             }
5382         }
5383
5384         return 0;
5385     };
5386
5387     return 1 if &$scan_config($conf);
5388
5389     undef $skip_drive;
5390
5391     foreach my $snapname (keys %{$conf->{snapshots}}) {
5392         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5393     }
5394
5395     return 0;
5396 }
5397
5398 sub update_disksize {
5399     my ($vmid, $conf, $volid_hash) = @_;
5400
5401     my $changes;
5402
5403     # used and unused disks
5404     my $referenced = {};
5405
5406     # Note: it is allowed to define multiple storages with same path (alias), so
5407     # we need to check both 'volid' and real 'path' (two different volid can point
5408     # to the same path).
5409
5410     my $referencedpath = {};
5411
5412     # update size info
5413     foreach my $opt (keys %$conf) {
5414         if (is_valid_drivename($opt)) {
5415             my $drive = parse_drive($opt, $conf->{$opt});
5416             my $volid = $drive->{file};
5417             next if !$volid;
5418
5419             $referenced->{$volid} = 1;
5420             if ($volid_hash->{$volid} &&
5421                 (my $path = $volid_hash->{$volid}->{path})) {
5422                 $referencedpath->{$path} = 1;
5423             }
5424
5425             next if drive_is_cdrom($drive);
5426             next if !$volid_hash->{$volid};
5427
5428             $drive->{size} = $volid_hash->{$volid}->{size};
5429             my $new = print_drive($vmid, $drive);
5430             if ($new ne $conf->{$opt}) {
5431                 $changes = 1;
5432                 $conf->{$opt} = $new;
5433             }
5434         }
5435     }
5436
5437     # remove 'unusedX' entry if volume is used
5438     foreach my $opt (keys %$conf) {
5439         next if $opt !~ m/^unused\d+$/;
5440         my $volid = $conf->{$opt};
5441         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5442         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5443             $changes = 1;
5444             delete $conf->{$opt};
5445         }
5446
5447         $referenced->{$volid} = 1;
5448         $referencedpath->{$path} = 1 if $path;
5449     }
5450
5451     foreach my $volid (sort keys %$volid_hash) {
5452         next if $volid =~ m/vm-$vmid-state-/;
5453         next if $referenced->{$volid};
5454         my $path = $volid_hash->{$volid}->{path};
5455         next if !$path; # just to be sure
5456         next if $referencedpath->{$path};
5457         $changes = 1;
5458         PVE::QemuConfig->add_unused_volume($conf, $volid);
5459         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5460     }
5461
5462     return $changes;
5463 }
5464
5465 sub rescan {
5466     my ($vmid, $nolock) = @_;
5467
5468     my $cfg = PVE::Storage::config();
5469
5470     my $volid_hash = scan_volids($cfg, $vmid);
5471
5472     my $updatefn =  sub {
5473         my ($vmid) = @_;
5474
5475         my $conf = PVE::QemuConfig->load_config($vmid);
5476
5477         PVE::QemuConfig->check_lock($conf);
5478
5479         my $vm_volids = {};
5480         foreach my $volid (keys %$volid_hash) {
5481             my $info = $volid_hash->{$volid};
5482             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5483         }
5484
5485         my $changes = update_disksize($vmid, $conf, $vm_volids);
5486
5487         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5488     };
5489
5490     if (defined($vmid)) {
5491         if ($nolock) {
5492             &$updatefn($vmid);
5493         } else {
5494             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5495         }
5496     } else {
5497         my $vmlist = config_list();
5498         foreach my $vmid (keys %$vmlist) {
5499             if ($nolock) {
5500                 &$updatefn($vmid);
5501             } else {
5502                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5503             }
5504         }
5505     }
5506 }
5507
5508 sub restore_vma_archive {
5509     my ($archive, $vmid, $user, $opts, $comp) = @_;
5510
5511     my $input = $archive eq '-' ? "<&STDIN" : undef;
5512     my $readfrom = $archive;
5513
5514     my $uncomp = '';
5515     if ($comp) {
5516         $readfrom = '-';
5517         my $qarchive = PVE::Tools::shellquote($archive);
5518         if ($comp eq 'gzip') {
5519             $uncomp = "zcat $qarchive|";
5520         } elsif ($comp eq 'lzop') {
5521             $uncomp = "lzop -d -c $qarchive|";
5522         } else {
5523             die "unknown compression method '$comp'\n";
5524         }
5525
5526     }
5527
5528     my $tmpdir = "/var/tmp/vzdumptmp$$";
5529     rmtree $tmpdir;
5530
5531     # disable interrupts (always do cleanups)
5532     local $SIG{INT} =
5533         local $SIG{TERM} =
5534         local $SIG{QUIT} =
5535         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5536
5537     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5538     POSIX::mkfifo($mapfifo, 0600);
5539     my $fifofh;
5540
5541     my $openfifo = sub {
5542         open($fifofh, '>', $mapfifo) || die $!;
5543     };
5544
5545     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5546
5547     my $oldtimeout;
5548     my $timeout = 5;
5549
5550     my $devinfo = {};
5551
5552     my $rpcenv = PVE::RPCEnvironment::get();
5553
5554     my $conffile = PVE::QemuConfig->config_file($vmid);
5555     my $tmpfn = "$conffile.$$.tmp";
5556
5557     # Note: $oldconf is undef if VM does not exists
5558     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5559     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5560
5561     my $print_devmap = sub {
5562         my $virtdev_hash = {};
5563
5564         my $cfgfn = "$tmpdir/qemu-server.conf";
5565
5566         # we can read the config - that is already extracted
5567         my $fh = IO::File->new($cfgfn, "r") ||
5568             "unable to read qemu-server.conf - $!\n";
5569
5570         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5571         if (-f $fwcfgfn) {
5572             my $pve_firewall_dir = '/etc/pve/firewall';
5573             mkdir $pve_firewall_dir; # make sure the dir exists
5574             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5575         }
5576
5577         while (defined(my $line = <$fh>)) {
5578             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5579                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5580                 die "archive does not contain data for drive '$virtdev'\n"
5581                     if !$devinfo->{$devname};
5582                 if (defined($opts->{storage})) {
5583                     $storeid = $opts->{storage} || 'local';
5584                 } elsif (!$storeid) {
5585                     $storeid = 'local';
5586                 }
5587                 $format = 'raw' if !$format;
5588                 $devinfo->{$devname}->{devname} = $devname;
5589                 $devinfo->{$devname}->{virtdev} = $virtdev;
5590                 $devinfo->{$devname}->{format} = $format;
5591                 $devinfo->{$devname}->{storeid} = $storeid;
5592
5593                 # check permission on storage
5594                 my $pool = $opts->{pool}; # todo: do we need that?
5595                 if ($user ne 'root@pam') {
5596                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5597                 }
5598
5599                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5600             }
5601         }
5602
5603         foreach my $devname (keys %$devinfo) {
5604             die "found no device mapping information for device '$devname'\n"
5605                 if !$devinfo->{$devname}->{virtdev};
5606         }
5607
5608         my $cfg = PVE::Storage::config();
5609
5610         # create empty/temp config
5611         if ($oldconf) {
5612             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5613             foreach_drive($oldconf, sub {
5614                 my ($ds, $drive) = @_;
5615
5616                 return if drive_is_cdrom($drive);
5617
5618                 my $volid = $drive->{file};
5619
5620                 return if !$volid || $volid =~ m|^/|;
5621
5622                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5623                 return if !$path || !$owner || ($owner != $vmid);
5624
5625                 # Note: only delete disk we want to restore
5626                 # other volumes will become unused
5627                 if ($virtdev_hash->{$ds}) {
5628                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5629                     if (my $err = $@) {
5630                         warn $err;
5631                     }
5632                 }
5633             });
5634
5635             # delete vmstate files
5636             # since after the restore we have no snapshots anymore
5637             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5638                 my $snap = $oldconf->{snapshots}->{$snapname};
5639                 if ($snap->{vmstate}) {
5640                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5641                     if (my $err = $@) {
5642                         warn $err;
5643                     }
5644                 }
5645             }
5646         }
5647
5648         my $map = {};
5649         foreach my $virtdev (sort keys %$virtdev_hash) {
5650             my $d = $virtdev_hash->{$virtdev};
5651             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5652             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5653
5654             # test if requested format is supported
5655             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5656             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5657             $d->{format} = $defFormat if !$supported;
5658
5659             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5660                                                   $d->{format}, undef, $alloc_size);
5661             print STDERR "new volume ID is '$volid'\n";
5662             $d->{volid} = $volid;
5663             my $path = PVE::Storage::path($cfg, $volid);
5664
5665             PVE::Storage::activate_volumes($cfg,[$volid]);
5666
5667             my $write_zeros = 1;
5668             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5669                 $write_zeros = 0;
5670             }
5671
5672             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5673
5674             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5675             $map->{$virtdev} = $volid;
5676         }
5677
5678         $fh->seek(0, 0) || die "seek failed - $!\n";
5679
5680         my $outfd = new IO::File ($tmpfn, "w") ||
5681             die "unable to write config for VM $vmid\n";
5682
5683         my $cookie = { netcount => 0 };
5684         while (defined(my $line = <$fh>)) {
5685             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5686         }
5687
5688         $fh->close();
5689         $outfd->close();
5690     };
5691
5692     eval {
5693         # enable interrupts
5694         local $SIG{INT} =
5695             local $SIG{TERM} =
5696             local $SIG{QUIT} =
5697             local $SIG{HUP} =
5698             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5699         local $SIG{ALRM} = sub { die "got timeout\n"; };
5700
5701         $oldtimeout = alarm($timeout);
5702
5703         my $parser = sub {
5704             my $line = shift;
5705
5706             print "$line\n";
5707
5708             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5709                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5710                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5711             } elsif ($line =~ m/^CTIME: /) {
5712                 # we correctly received the vma config, so we can disable
5713                 # the timeout now for disk allocation (set to 10 minutes, so
5714                 # that we always timeout if something goes wrong)
5715                 alarm(600);
5716                 &$print_devmap();
5717                 print $fifofh "done\n";
5718                 my $tmp = $oldtimeout || 0;
5719                 $oldtimeout = undef;
5720                 alarm($tmp);
5721                 close($fifofh);
5722             }
5723         };
5724
5725         print "restore vma archive: $cmd\n";
5726         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5727     };
5728     my $err = $@;
5729
5730     alarm($oldtimeout) if $oldtimeout;
5731
5732     my $vollist = [];
5733     foreach my $devname (keys %$devinfo) {
5734         my $volid = $devinfo->{$devname}->{volid};
5735         push @$vollist, $volid if $volid;
5736     }
5737
5738     my $cfg = PVE::Storage::config();
5739     PVE::Storage::deactivate_volumes($cfg, $vollist);
5740
5741     unlink $mapfifo;
5742
5743     if ($err) {
5744         rmtree $tmpdir;
5745         unlink $tmpfn;
5746
5747         foreach my $devname (keys %$devinfo) {
5748             my $volid = $devinfo->{$devname}->{volid};
5749             next if !$volid;
5750             eval {
5751                 if ($volid =~ m|^/|) {
5752                     unlink $volid || die 'unlink failed\n';
5753                 } else {
5754                     PVE::Storage::vdisk_free($cfg, $volid);
5755                 }
5756                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5757             };
5758             print STDERR "unable to cleanup '$volid' - $@" if $@;
5759         }
5760         die $err;
5761     }
5762
5763     rmtree $tmpdir;
5764
5765     rename($tmpfn, $conffile) ||
5766         die "unable to commit configuration file '$conffile'\n";
5767
5768     PVE::Cluster::cfs_update(); # make sure we read new file
5769
5770     eval { rescan($vmid, 1); };
5771     warn $@ if $@;
5772 }
5773
5774 sub restore_tar_archive {
5775     my ($archive, $vmid, $user, $opts) = @_;
5776
5777     if ($archive ne '-') {
5778         my $firstfile = tar_archive_read_firstfile($archive);
5779         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5780             if $firstfile ne 'qemu-server.conf';
5781     }
5782
5783     my $storecfg = PVE::Storage::config();
5784
5785     # destroy existing data - keep empty config
5786     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5787     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5788
5789     my $tocmd = "/usr/lib/qemu-server/qmextract";
5790
5791     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5792     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5793     $tocmd .= ' --prealloc' if $opts->{prealloc};
5794     $tocmd .= ' --info' if $opts->{info};
5795
5796     # tar option "xf" does not autodetect compression when read from STDIN,
5797     # so we pipe to zcat
5798     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5799         PVE::Tools::shellquote("--to-command=$tocmd");
5800
5801     my $tmpdir = "/var/tmp/vzdumptmp$$";
5802     mkpath $tmpdir;
5803
5804     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5805     local $ENV{VZDUMP_VMID} = $vmid;
5806     local $ENV{VZDUMP_USER} = $user;
5807
5808     my $conffile = PVE::QemuConfig->config_file($vmid);
5809     my $tmpfn = "$conffile.$$.tmp";
5810
5811     # disable interrupts (always do cleanups)
5812     local $SIG{INT} =
5813         local $SIG{TERM} =
5814         local $SIG{QUIT} =
5815         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5816
5817     eval {
5818         # enable interrupts
5819         local $SIG{INT} =
5820             local $SIG{TERM} =
5821             local $SIG{QUIT} =
5822             local $SIG{HUP} =
5823             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5824
5825         if ($archive eq '-') {
5826             print "extracting archive from STDIN\n";
5827             run_command($cmd, input => "<&STDIN");
5828         } else {
5829             print "extracting archive '$archive'\n";
5830             run_command($cmd);
5831         }
5832
5833         return if $opts->{info};
5834
5835         # read new mapping
5836         my $map = {};
5837         my $statfile = "$tmpdir/qmrestore.stat";
5838         if (my $fd = IO::File->new($statfile, "r")) {
5839             while (defined (my $line = <$fd>)) {
5840                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5841                     $map->{$1} = $2 if $1;
5842                 } else {
5843                     print STDERR "unable to parse line in statfile - $line\n";
5844                 }
5845             }
5846             $fd->close();
5847         }
5848
5849         my $confsrc = "$tmpdir/qemu-server.conf";
5850
5851         my $srcfd = new IO::File($confsrc, "r") ||
5852             die "unable to open file '$confsrc'\n";
5853
5854         my $outfd = new IO::File ($tmpfn, "w") ||
5855             die "unable to write config for VM $vmid\n";
5856
5857         my $cookie = { netcount => 0 };
5858         while (defined (my $line = <$srcfd>)) {
5859             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5860         }
5861
5862         $srcfd->close();
5863         $outfd->close();
5864     };
5865     my $err = $@;
5866
5867     if ($err) {
5868
5869         unlink $tmpfn;
5870
5871         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5872
5873         die $err;
5874     }
5875
5876     rmtree $tmpdir;
5877
5878     rename $tmpfn, $conffile ||
5879         die "unable to commit configuration file '$conffile'\n";
5880
5881     PVE::Cluster::cfs_update(); # make sure we read new file
5882
5883     eval { rescan($vmid, 1); };
5884     warn $@ if $@;
5885 };
5886
5887 sub foreach_storage_used_by_vm {
5888     my ($conf, $func) = @_;
5889
5890     my $sidhash = {};
5891
5892     foreach_drive($conf, sub {
5893         my ($ds, $drive) = @_;
5894         return if drive_is_cdrom($drive);
5895
5896         my $volid = $drive->{file};
5897
5898         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5899         $sidhash->{$sid} = $sid if $sid;
5900     });
5901
5902     foreach my $sid (sort keys %$sidhash) {
5903         &$func($sid);
5904     }
5905 }
5906
5907 sub do_snapshots_with_qemu {
5908     my ($storecfg, $volid) = @_;
5909
5910     my $storage_name = PVE::Storage::parse_volume_id($volid);
5911
5912     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5913         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5914         return 1;
5915     }
5916
5917     if ($volid =~ m/\.(qcow2|qed)$/){
5918         return 1;
5919     }
5920
5921     return undef;
5922 }
5923
5924 sub qga_check_running {
5925     my ($vmid) = @_;
5926
5927     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5928     if ($@) {
5929         warn "Qemu Guest Agent is not running - $@";
5930         return 0;
5931     }
5932     return 1;
5933 }
5934
5935 sub template_create {
5936     my ($vmid, $conf, $disk) = @_;
5937
5938     my $storecfg = PVE::Storage::config();
5939
5940     foreach_drive($conf, sub {
5941         my ($ds, $drive) = @_;
5942
5943         return if drive_is_cdrom($drive);
5944         return if $disk && $ds ne $disk;
5945
5946         my $volid = $drive->{file};
5947         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5948
5949         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5950         $drive->{file} = $voliddst;
5951         $conf->{$ds} = print_drive($vmid, $drive);
5952         PVE::QemuConfig->write_config($vmid, $conf);
5953     });
5954 }
5955
5956 sub qemu_img_convert {
5957     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5958
5959     my $storecfg = PVE::Storage::config();
5960     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5961     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5962
5963     if ($src_storeid && $dst_storeid) {
5964
5965         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5966
5967         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5968         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5969
5970         my $src_format = qemu_img_format($src_scfg, $src_volname);
5971         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5972
5973         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5974         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5975
5976         my $cmd = [];
5977         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5978         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5979         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5980         if ($is_zero_initialized) {
5981             push @$cmd, "zeroinit:$dst_path";
5982         } else {
5983             push @$cmd, $dst_path;
5984         }
5985
5986         my $parser = sub {
5987             my $line = shift;
5988             if($line =~ m/\((\S+)\/100\%\)/){
5989                 my $percent = $1;
5990                 my $transferred = int($size * $percent / 100);
5991                 my $remaining = $size - $transferred;
5992
5993                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5994             }
5995
5996         };
5997
5998         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5999         my $err = $@;
6000         die "copy failed: $err" if $err;
6001     }
6002 }
6003
6004 sub qemu_img_format {
6005     my ($scfg, $volname) = @_;
6006
6007     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6008         return $1;
6009     } else {
6010         return "raw";
6011     }
6012 }
6013
6014 sub qemu_drive_mirror {
6015     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6016
6017     $jobs = {} if !$jobs;
6018
6019     my $qemu_target;
6020     my $format;
6021     $jobs->{"drive-$drive"} = {};
6022
6023     if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
6024         my $server = $1;
6025         my $port = $2;
6026         my $exportname = $3;
6027
6028         $format = "nbd";
6029         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
6030         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
6031         my $cmd = ['socat', '-T30', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=5"];
6032
6033         my $pid = fork();
6034         if (!defined($pid)) {
6035             die "forking socat tunnel failed\n";
6036         } elsif ($pid == 0) {
6037             exec(@$cmd);
6038             warn "exec failed: $!\n";
6039             POSIX::_exit(-1);
6040         }
6041         $jobs->{"drive-$drive"}->{pid} = $pid;
6042
6043         my $timeout = 0;
6044         while (!-S $unixsocket) {
6045             die "nbd connection helper timed out\n"
6046                 if $timeout++ > 5;
6047             sleep 1;
6048         }
6049     } else {
6050         my $storecfg = PVE::Storage::config();
6051         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6052
6053         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6054
6055         $format = qemu_img_format($dst_scfg, $dst_volname);
6056
6057         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6058
6059         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6060     }
6061
6062     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6063     $opts->{format} = $format if $format;
6064
6065     print "drive mirror is starting for drive-$drive\n";
6066
6067     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6068
6069     if (my $err = $@) {
6070         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6071         die "mirroring error: $err";
6072     }
6073
6074     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6075 }
6076
6077 sub qemu_drive_mirror_monitor {
6078     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6079
6080     eval {
6081         my $err_complete = 0;
6082
6083         while (1) {
6084             die "storage migration timed out\n" if $err_complete > 300;
6085
6086             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6087
6088             my $running_mirror_jobs = {};
6089             foreach my $stat (@$stats) {
6090                 next if $stat->{type} ne 'mirror';
6091                 $running_mirror_jobs->{$stat->{device}} = $stat;
6092             }
6093
6094             my $readycounter = 0;
6095
6096             foreach my $job (keys %$jobs) {
6097
6098                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6099                     print "$job : finished\n";
6100                     delete $jobs->{$job};
6101                     next;
6102                 }
6103
6104                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6105
6106                 my $busy = $running_mirror_jobs->{$job}->{busy};
6107                 my $ready = $running_mirror_jobs->{$job}->{ready};
6108                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6109                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6110                     my $remaining = $total - $transferred;
6111                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6112
6113                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6114                 }
6115
6116                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6117             }
6118
6119             last if scalar(keys %$jobs) == 0;
6120
6121             if ($readycounter == scalar(keys %$jobs)) {
6122                 print "all mirroring jobs are ready \n";
6123                 last if $skipcomplete; #do the complete later
6124
6125                 if ($vmiddst && $vmiddst != $vmid) {
6126                     my $agent_running = $qga && qga_check_running($vmid);
6127                     if ($agent_running) {
6128                         print "freeze filesystem\n";
6129                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6130                     } else {
6131                         print "suspend vm\n";
6132                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6133                     }
6134
6135                     # if we clone a disk for a new target vm, we don't switch the disk
6136                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6137
6138                     if ($agent_running) {
6139                         print "unfreeze filesystem\n";
6140                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6141                     } else {
6142                         print "resume vm\n";
6143                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6144                     }
6145
6146                     last;
6147                 } else {
6148
6149                     foreach my $job (keys %$jobs) {
6150                         # try to switch the disk if source and destination are on the same guest
6151                         print "$job: Completing block job...\n";
6152
6153                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6154                         if ($@ =~ m/cannot be completed/) {
6155                             print "$job: Block job cannot be completed, try again.\n";
6156                             $err_complete++;
6157                         }else {
6158                             print "$job: Completed successfully.\n";
6159                             $jobs->{$job}->{complete} = 1;
6160                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6161                         }
6162                     }
6163                 }
6164             }
6165             sleep 1;
6166         }
6167     };
6168     my $err = $@;
6169
6170     if ($err) {
6171         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6172         die "mirroring error: $err";
6173     }
6174
6175 }
6176
6177 sub qemu_blockjobs_cancel {
6178     my ($vmid, $jobs) = @_;
6179
6180     foreach my $job (keys %$jobs) {
6181         print "$job: Cancelling block job\n";
6182         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6183         $jobs->{$job}->{cancel} = 1;
6184     }
6185
6186     while (1) {
6187         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6188
6189         my $running_jobs = {};
6190         foreach my $stat (@$stats) {
6191             $running_jobs->{$stat->{device}} = $stat;
6192         }
6193
6194         foreach my $job (keys %$jobs) {
6195
6196             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6197                 print "$job: Done.\n";
6198                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6199                 delete $jobs->{$job};
6200             }
6201         }
6202
6203         last if scalar(keys %$jobs) == 0;
6204
6205         sleep 1;
6206     }
6207 }
6208
6209 sub qemu_blockjobs_finish_tunnel {
6210    my ($vmid, $job, $cpid) = @_;
6211
6212    return if !$cpid;
6213
6214    for (my $i = 1; $i < 20; $i++) {
6215         my $waitpid = waitpid($cpid, WNOHANG);
6216         last if (defined($waitpid) && ($waitpid == $cpid));
6217
6218         if ($i == 10) {
6219             kill(15, $cpid);
6220         } elsif ($i >= 15) {
6221             kill(9, $cpid);
6222         }
6223         sleep (1);
6224     }
6225     unlink "/run/qemu-server/$vmid.mirror-$job";
6226 }
6227
6228 sub clone_disk {
6229     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6230         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6231
6232     my $newvolid;
6233
6234     if (!$full) {
6235         print "create linked clone of drive $drivename ($drive->{file})\n";
6236         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6237         push @$newvollist, $newvolid;
6238     } else {
6239
6240         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6241         $storeid = $storage if $storage;
6242
6243         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6244         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6245
6246         print "create full clone of drive $drivename ($drive->{file})\n";
6247         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, undef, ($size/1024));
6248         push @$newvollist, $newvolid;
6249
6250         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6251
6252         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6253         if (!$running || $snapname) {
6254             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6255         } else {
6256
6257             my $kvmver = get_running_qemu_version ($vmid);
6258             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6259                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6260                     if $drive->{iothread};
6261             }
6262
6263             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6264         }
6265     }
6266
6267     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6268
6269     my $disk = $drive;
6270     $disk->{format} = undef;
6271     $disk->{file} = $newvolid;
6272     $disk->{size} = $size;
6273
6274     return $disk;
6275 }
6276
6277 # this only works if VM is running
6278 sub get_current_qemu_machine {
6279     my ($vmid) = @_;
6280
6281     my $cmd = { execute => 'query-machines', arguments => {} };
6282     my $res = vm_qmp_command($vmid, $cmd);
6283
6284     my ($current, $default);
6285     foreach my $e (@$res) {
6286         $default = $e->{name} if $e->{'is-default'};
6287         $current = $e->{name} if $e->{'is-current'};
6288     }
6289
6290     # fallback to the default machine if current is not supported by qemu
6291     return $current || $default || 'pc';
6292 }
6293
6294 sub get_running_qemu_version {
6295     my ($vmid) = @_;
6296     my $cmd = { execute => 'query-version', arguments => {} };
6297     my $res = vm_qmp_command($vmid, $cmd);
6298     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6299 }
6300
6301 sub qemu_machine_feature_enabled {
6302     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6303
6304     my $current_major;
6305     my $current_minor;
6306
6307     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6308
6309         $current_major = $3;
6310         $current_minor = $4;
6311
6312     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6313
6314         $current_major = $1;
6315         $current_minor = $2;
6316     }
6317
6318     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6319
6320
6321 }
6322
6323 sub qemu_machine_pxe {
6324     my ($vmid, $conf, $machine) = @_;
6325
6326     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6327
6328     foreach my $opt (keys %$conf) {
6329         next if $opt !~ m/^net(\d+)$/;
6330         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6331         next if !$net;
6332         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6333         return $machine.".pxe" if $romfile =~ m/pxe/;
6334         last;
6335     }
6336
6337     return $machine;
6338 }
6339
6340 sub qemu_use_old_bios_files {
6341     my ($machine_type) = @_;
6342
6343     return if !$machine_type;
6344
6345     my $use_old_bios_files = undef;
6346
6347     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6348         $machine_type = $1;
6349         $use_old_bios_files = 1;
6350     } else {
6351         my $kvmver = kvm_user_version();
6352         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6353         # load new efi bios files on migration. So this hack is required to allow
6354         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6355         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6356         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6357     }
6358
6359     return ($use_old_bios_files, $machine_type);
6360 }
6361
6362 sub create_efidisk {
6363     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6364
6365     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6366
6367     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6368     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6369     PVE::Storage::activate_volumes($storecfg, [$volid]);
6370
6371     my $path = PVE::Storage::path($storecfg, $volid);
6372     eval {
6373         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6374     };
6375     die "Copying EFI vars image failed: $@" if $@;
6376
6377     return ($volid, $vars_size);
6378 }
6379
6380 sub lspci {
6381
6382     my $devices = {};
6383
6384     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6385             my (undef, $id, $function) = @_;
6386             my $res = { id => $id, function => $function};
6387             push @{$devices->{$id}}, $res;
6388     });
6389
6390     # Entries should be sorted by functions.
6391     foreach my $id (keys %$devices) {
6392         my $dev = $devices->{$id};
6393         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6394     }
6395
6396     return $devices;
6397 }
6398
6399 sub vm_iothreads_list {
6400     my ($vmid) = @_;
6401
6402     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6403
6404     my $iothreads = {};
6405     foreach my $iothread (@$res) {
6406         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6407     }
6408
6409     return $iothreads;
6410 }
6411
6412 sub scsihw_infos {
6413     my ($conf, $drive) = @_;
6414
6415     my $maxdev = 0;
6416
6417     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6418         $maxdev = 7;
6419     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6420         $maxdev = 1;
6421     } else {
6422         $maxdev = 256;
6423     }
6424
6425     my $controller = int($drive->{index} / $maxdev);
6426     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6427
6428     return ($maxdev, $controller, $controller_prefix);
6429 }
6430
6431 sub add_hyperv_enlightenments {
6432     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6433
6434     return if $winversion < 6;
6435     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6436
6437     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6438
6439     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6440         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6441         push @$cpuFlags , 'hv_vapic';
6442         push @$cpuFlags , 'hv_time';
6443     } else {
6444         push @$cpuFlags , 'hv_spinlocks=0xffff';
6445     }
6446
6447     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6448         push @$cpuFlags , 'hv_reset';
6449         push @$cpuFlags , 'hv_vpindex';
6450         push @$cpuFlags , 'hv_runtime';
6451     }
6452
6453     if ($winversion >= 7) {
6454         push @$cpuFlags , 'hv_relaxed';
6455     }
6456 }
6457
6458 sub windows_version {
6459     my ($ostype) = @_;
6460
6461     return 0 if !$ostype;
6462
6463     my $winversion = 0;
6464
6465     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6466         $winversion = 5;
6467     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6468         $winversion = 6;
6469     } elsif ($ostype =~ m/^win(\d+)$/) {
6470         $winversion = $1;
6471     }
6472
6473     return $winversion;
6474 }
6475
6476 sub resolve_dst_disk_format {
6477         my ($storecfg, $storeid, $src_volname, $format) = @_;
6478         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6479
6480         if (!$format) {
6481             # if no target format is specified, use the source disk format as hint
6482             if ($src_volname) {
6483                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6484                 $format = qemu_img_format($scfg, $src_volname);
6485             } else {
6486                 return $defFormat;
6487             }
6488         }
6489
6490         # test if requested format is supported - else use default
6491         my $supported = grep { $_ eq $format } @$validFormats;
6492         $format = $defFormat if !$supported;
6493         return $format;
6494 }
6495
6496 sub resolve_first_disk {
6497     my $conf = shift;
6498     my @disks = PVE::QemuServer::valid_drive_names();
6499     my $firstdisk;
6500     foreach my $ds (reverse @disks) {
6501         next if !$conf->{$ds};
6502         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6503         next if PVE::QemuServer::drive_is_cdrom($disk);
6504         $firstdisk = $ds;
6505     }
6506     return $firstdisk;
6507 }
6508
6509 sub generate_smbios1_uuid {
6510     my ($uuid, $uuid_str);
6511     UUID::generate($uuid);
6512     UUID::unparse($uuid, $uuid_str);
6513     return "uuid=$uuid_str";
6514 }
6515
6516 # bash completion helper
6517
6518 sub complete_backup_archives {
6519     my ($cmdname, $pname, $cvalue) = @_;
6520
6521     my $cfg = PVE::Storage::config();
6522
6523     my $storeid;
6524
6525     if ($cvalue =~ m/^([^:]+):/) {
6526         $storeid = $1;
6527     }
6528
6529     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6530
6531     my $res = [];
6532     foreach my $id (keys %$data) {
6533         foreach my $item (@{$data->{$id}}) {
6534             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6535             push @$res, $item->{volid} if defined($item->{volid});
6536         }
6537     }
6538
6539     return $res;
6540 }
6541
6542 my $complete_vmid_full = sub {
6543     my ($running) = @_;
6544
6545     my $idlist = vmstatus();
6546
6547     my $res = [];
6548
6549     foreach my $id (keys %$idlist) {
6550         my $d = $idlist->{$id};
6551         if (defined($running)) {
6552             next if $d->{template};
6553             next if $running && $d->{status} ne 'running';
6554             next if !$running && $d->{status} eq 'running';
6555         }
6556         push @$res, $id;
6557
6558     }
6559     return $res;
6560 };
6561
6562 sub complete_vmid {
6563     return &$complete_vmid_full();
6564 }
6565
6566 sub complete_vmid_stopped {
6567     return &$complete_vmid_full(0);
6568 }
6569
6570 sub complete_vmid_running {
6571     return &$complete_vmid_full(1);
6572 }
6573
6574 sub complete_storage {
6575
6576     my $cfg = PVE::Storage::config();
6577     my $ids = $cfg->{ids};
6578
6579     my $res = [];
6580     foreach my $sid (keys %$ids) {
6581         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6582         next if !$ids->{$sid}->{content}->{images};
6583         push @$res, $sid;
6584     }
6585
6586     return $res;
6587 }
6588
6589 sub nbd_stop {
6590     my ($vmid) = @_;
6591
6592     vm_mon_cmd($vmid, 'nbd-server-stop');
6593 }
6594
6595 1;