PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42
  43 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  44
  45 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  46
  47 # Note about locking: we use flock on the config file protect
  48 # against concurent actions.
  49 # Aditionaly, we have a 'lock' setting in the config file. This
  50 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  51 # allowed when such lock is set. But you can ignore this kind of
  52 # lock with the --skiplock flag.
  53
  54 cfs_register_file('/qemu-server/',
  55                   \&parse_vm_config,
  56                   \&write_vm_config);
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  72     type => 'string',
  73     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  74     description => "The drive's backing file's data format.",
  75     optional => 1,
  76 });
  77
  78 #no warnings 'redefine';
  79
  80 sub cgroups_write {
  81    my ($controller, $vmid, $option, $value) = @_;
  82
  83    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  84    PVE::ProcFSTools::write_proc_entry($path, $value);
  85
  86 }
  87
  88 my $nodename = PVE::INotify::nodename();
  89
  90 mkdir "/etc/pve/nodes/$nodename";
  91 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  92 mkdir $confdir;
  93
  94 my $var_run_tmpdir = "/var/run/qemu-server";
  95 mkdir $var_run_tmpdir;
  96
  97 my $lock_dir = "/var/lock/qemu-server";
  98 mkdir $lock_dir;
  99
 100 my $pcisysfs = "/sys/bus/pci";
 101
 102 my $cpu_vendor_list = {
 103     # Intel CPUs
 104     486 => 'GenuineIntel',
 105     pentium => 'GenuineIntel',
 106     pentium2  => 'GenuineIntel',
 107     pentium3  => 'GenuineIntel',
 108     coreduo => 'GenuineIntel',
 109     core2duo => 'GenuineIntel',
 110     Conroe  => 'GenuineIntel',
 111     Penryn  => 'GenuineIntel',
 112     Nehalem  => 'GenuineIntel',
 113     'Nehalem-IBRS'  => 'GenuineIntel',
 114     Westmere => 'GenuineIntel',
 115     'Westmere-IBRS' => 'GenuineIntel',
 116     SandyBridge => 'GenuineIntel',
 117     'SandyBridge-IBRS' => 'GenuineIntel',
 118     IvyBridge => 'GenuineIntel',
 119     'IvyBridge-IBRS' => 'GenuineIntel',
 120     Haswell => 'GenuineIntel',
 121     'Haswell-IBRS' => 'GenuineIntel',
 122     'Haswell-noTSX' => 'GenuineIntel',
 123     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 124     Broadwell => 'GenuineIntel',
 125     'Broadwell-IBRS' => 'GenuineIntel',
 126     'Broadwell-noTSX' => 'GenuineIntel',
 127     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 128     'Skylake-Client' => 'GenuineIntel',
 129     'Skylake-Client-IBRS' => 'GenuineIntel',
 130     'Skylake-Server' => 'GenuineIntel',
 131     'Skylake-Server-IBRS' => 'GenuineIntel',
 132
 133     # AMD CPUs
 134     athlon => 'AuthenticAMD',
 135     phenom  => 'AuthenticAMD',
 136     Opteron_G1  => 'AuthenticAMD',
 137     Opteron_G2  => 'AuthenticAMD',
 138     Opteron_G3  => 'AuthenticAMD',
 139     Opteron_G4  => 'AuthenticAMD',
 140     Opteron_G5  => 'AuthenticAMD',
 141
 142     # generic types, use vendor from host node
 143     host => 'default',
 144     kvm32 => 'default',
 145     kvm64 => 'default',
 146     qemu32 => 'default',
 147     qemu64 => 'default',
 148     max => 'default',
 149 };
 150
 151 my $cpu_flag = qr/[+-](pcid|spec-ctrl)/;
 152
 153 my $cpu_fmt = {
 154     cputype => {
 155         description => "Emulated CPU type.",
 156         type => 'string',
 157         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 158         default => 'kvm64',
 159         default_key => 1,
 160     },
 161     hidden => {
 162         description => "Do not identify as a KVM virtual machine.",
 163         type => 'boolean',
 164         optional => 1,
 165         default => 0
 166     },
 167     flags => {
 168         description => "List of additional CPU flags separated by ';'."
 169                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 170                      . " Currently supported flags: 'pcid', 'spec-ctrl'.",
 171         format_description => '+FLAG[;-FLAG...]',
 172         type => 'string',
 173         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 174         optional => 1,
 175     },
 176 };
 177
 178 my $watchdog_fmt = {
 179     model => {
 180         default_key => 1,
 181         type => 'string',
 182         enum => [qw(i6300esb ib700)],
 183         description => "Watchdog type to emulate.",
 184         default => 'i6300esb',
 185         optional => 1,
 186     },
 187     action => {
 188         type => 'string',
 189         enum => [qw(reset shutdown poweroff pause debug none)],
 190         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 191         optional => 1,
 192     },
 193 };
 194 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 195
 196 my $confdesc = {
 197     onboot => {
 198         optional => 1,
 199         type => 'boolean',
 200         description => "Specifies whether a VM will be started during system bootup.",
 201         default => 0,
 202     },
 203     autostart => {
 204         optional => 1,
 205         type => 'boolean',
 206         description => "Automatic restart after crash (currently ignored).",
 207         default => 0,
 208     },
 209     hotplug => {
 210         optional => 1,
 211         type => 'string', format => 'pve-hotplug-features',
 212         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 213         default => 'network,disk,usb',
 214     },
 215     reboot => {
 216         optional => 1,
 217         type => 'boolean',
 218         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 219         default => 1,
 220     },
 221     lock => {
 222         optional => 1,
 223         type => 'string',
 224         description => "Lock/unlock the VM.",
 225         enum => [qw(migrate backup snapshot rollback)],
 226     },
 227     cpulimit => {
 228         optional => 1,
 229         type => 'number',
 230         description => "Limit of CPU usage.",
 231         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 232         minimum => 0,
 233         maximum => 128,
 234         default => 0,
 235     },
 236     cpuunits => {
 237         optional => 1,
 238         type => 'integer',
 239         description => "CPU weight for a VM.",
 240         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 241         minimum => 2,
 242         maximum => 262144,
 243         default => 1024,
 244     },
 245     memory => {
 246         optional => 1,
 247         type => 'integer',
 248         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 249         minimum => 16,
 250         default => 512,
 251     },
 252     balloon => {
 253         optional => 1,
 254         type => 'integer',
 255         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 256         minimum => 0,
 257     },
 258     shares => {
 259         optional => 1,
 260         type => 'integer',
 261         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 262         minimum => 0,
 263         maximum => 50000,
 264         default => 1000,
 265     },
 266     keyboard => {
 267         optional => 1,
 268         type => 'string',
 269         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.".
 270                        "It should not be necessary to set it.",
 271         enum => PVE::Tools::kvmkeymaplist(),
 272         default => undef,
 273     },
 274     name => {
 275         optional => 1,
 276         type => 'string', format => 'dns-name',
 277         description => "Set a name for the VM. Only used on the configuration web interface.",
 278     },
 279     scsihw => {
 280         optional => 1,
 281         type => 'string',
 282         description => "SCSI controller model",
 283         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 284         default => 'lsi',
 285     },
 286     description => {
 287         optional => 1,
 288         type => 'string',
 289         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 290     },
 291     ostype => {
 292         optional => 1,
 293         type => 'string',
 294         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 295         description => "Specify guest operating system.",
 296         verbose_description => <<EODESC,
 297 Specify guest operating system. This is used to enable special
 298 optimization/features for specific operating systems:
 299
 300 [horizontal]
 301 other;; unspecified OS
 302 wxp;; Microsoft Windows XP
 303 w2k;; Microsoft Windows 2000
 304 w2k3;; Microsoft Windows 2003
 305 w2k8;; Microsoft Windows 2008
 306 wvista;; Microsoft Windows Vista
 307 win7;; Microsoft Windows 7
 308 win8;; Microsoft Windows 8/2012/2012r2
 309 win10;; Microsoft Windows 10/2016
 310 l24;; Linux 2.4 Kernel
 311 l26;; Linux 2.6/3.X Kernel
 312 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 313 EODESC
 314     },
 315     boot => {
 316         optional => 1,
 317         type => 'string',
 318         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 319         pattern => '[acdn]{1,4}',
 320         default => 'cdn',
 321     },
 322     bootdisk => {
 323         optional => 1,
 324         type => 'string', format => 'pve-qm-bootdisk',
 325         description => "Enable booting from specified disk.",
 326         pattern => '(ide|sata|scsi|virtio)\d+',
 327     },
 328     smp => {
 329         optional => 1,
 330         type => 'integer',
 331         description => "The number of CPUs. Please use option -sockets instead.",
 332         minimum => 1,
 333         default => 1,
 334     },
 335     sockets => {
 336         optional => 1,
 337         type => 'integer',
 338         description => "The number of CPU sockets.",
 339         minimum => 1,
 340         default => 1,
 341     },
 342     cores => {
 343         optional => 1,
 344         type => 'integer',
 345         description => "The number of cores per socket.",
 346         minimum => 1,
 347         default => 1,
 348     },
 349     numa => {
 350         optional => 1,
 351         type => 'boolean',
 352         description => "Enable/disable NUMA.",
 353         default => 0,
 354     },
 355     hugepages => {
 356         optional => 1,
 357         type => 'string',
 358         description => "Enable/disable hugepages memory.",
 359         enum => [qw(any 2 1024)],
 360     },
 361     vcpus => {
 362         optional => 1,
 363         type => 'integer',
 364         description => "Number of hotplugged vcpus.",
 365         minimum => 1,
 366         default => 0,
 367     },
 368     acpi => {
 369         optional => 1,
 370         type => 'boolean',
 371         description => "Enable/disable ACPI.",
 372         default => 1,
 373     },
 374     agent => {
 375         optional => 1,
 376         type => 'boolean',
 377         description => "Enable/disable Qemu GuestAgent.",
 378         default => 0,
 379     },
 380     kvm => {
 381         optional => 1,
 382         type => 'boolean',
 383         description => "Enable/disable KVM hardware virtualization.",
 384         default => 1,
 385     },
 386     tdf => {
 387         optional => 1,
 388         type => 'boolean',
 389         description => "Enable/disable time drift fix.",
 390         default => 0,
 391     },
 392     localtime => {
 393         optional => 1,
 394         type => 'boolean',
 395         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 396     },
 397     freeze => {
 398         optional => 1,
 399         type => 'boolean',
 400         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 401     },
 402     vga => {
 403         optional => 1,
 404         type => 'string',
 405         description => "Select the VGA type.",
 406         verbose_description => "Select the VGA type. If you want to use high resolution" .
 407             " modes (>= 1280x1024x16) then you should use the options " .
 408             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 409             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 410             "display sever. For win* OS you can select how many independent " .
 411             "displays you want, Linux guests can add displays them self. " .
 412             "You can also run without any graphic card, using a serial device" .
 413             " as terminal.",
 414         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 415     },
 416     watchdog => {
 417         optional => 1,
 418         type => 'string', format => 'pve-qm-watchdog',
 419         description => "Create a virtual hardware watchdog device.",
 420         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 421             " (by a guest action), the watchdog must be periodically polled " .
 422             "by an agent inside the guest or else the watchdog will reset " .
 423             "the guest (or execute the respective action specified)",
 424     },
 425     startdate => {
 426         optional => 1,
 427         type => 'string',
 428         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 429         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 430         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 431         default => 'now',
 432     },
 433     startup =>  get_standard_option('pve-startup-order'),
 434     template => {
 435         optional => 1,
 436         type => 'boolean',
 437         description => "Enable/disable Template.",
 438         default => 0,
 439     },
 440     args => {
 441         optional => 1,
 442         type => 'string',
 443         description => "Arbitrary arguments passed to kvm.",
 444         verbose_description => <<EODESCR,
 445 Arbitrary arguments passed to kvm, for example:
 446
 447 args: -no-reboot -no-hpet
 448
 449 NOTE: this option is for experts only.
 450 EODESCR
 451     },
 452     tablet => {
 453         optional => 1,
 454         type => 'boolean',
 455         default => 1,
 456         description => "Enable/disable the USB tablet device.",
 457         verbose_description => "Enable/disable the USB tablet device. This device is " .
 458             "usually needed to allow absolute mouse positioning with VNC. " .
 459             "Else the mouse runs out of sync with normal VNC clients. " .
 460             "If you're running lots of console-only guests on one host, " .
 461             "you may consider disabling this to save some context switches. " .
 462             "This is turned off by default if you use spice (-vga=qxl).",
 463     },
 464     migrate_speed => {
 465         optional => 1,
 466         type => 'integer',
 467         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 468         minimum => 0,
 469         default => 0,
 470     },
 471     migrate_downtime => {
 472         optional => 1,
 473         type => 'number',
 474         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 475         minimum => 0,
 476         default => 0.1,
 477     },
 478     cdrom => {
 479         optional => 1,
 480         type => 'string', format => 'pve-qm-ide',
 481         typetext => '<volume>',
 482         description => "This is an alias for option -ide2",
 483     },
 484     cpu => {
 485         optional => 1,
 486         description => "Emulated CPU type.",
 487         type => 'string',
 488         format => $cpu_fmt,
 489     },
 490     parent => get_standard_option('pve-snapshot-name', {
 491         optional => 1,
 492         description => "Parent snapshot name. This is used internally, and should not be modified.",
 493     }),
 494     snaptime => {
 495         optional => 1,
 496         description => "Timestamp for snapshots.",
 497         type => 'integer',
 498         minimum => 0,
 499     },
 500     vmstate => {
 501         optional => 1,
 502         type => 'string', format => 'pve-volume-id',
 503         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 504     },
 505     vmstatestorage => get_standard_option('pve-storage-id', {
 506         description => "Default storage for VM state volumes/files.",
 507         optional => 1,
 508     }),
 509     machine => {
 510         description => "Specific the Qemu machine type.",
 511         type => 'string',
 512         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 513         maxLength => 40,
 514         optional => 1,
 515     },
 516     smbios1 => {
 517         description => "Specify SMBIOS type 1 fields.",
 518         type => 'string', format => 'pve-qm-smbios1',
 519         maxLength => 256,
 520         optional => 1,
 521     },
 522     protection => {
 523         optional => 1,
 524         type => 'boolean',
 525         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 526         default => 0,
 527     },
 528     bios => {
 529         optional => 1,
 530         type => 'string',
 531         enum => [ qw(seabios ovmf) ],
 532         description => "Select BIOS implementation.",
 533         default => 'seabios',
 534     },
 535 };
 536
 537 # what about other qemu settings ?
 538 #cpu => 'string',
 539 #machine => 'string',
 540 #fda => 'file',
 541 #fdb => 'file',
 542 #mtdblock => 'file',
 543 #sd => 'file',
 544 #pflash => 'file',
 545 #snapshot => 'bool',
 546 #bootp => 'file',
 547 ##tftp => 'dir',
 548 ##smb => 'dir',
 549 #kernel => 'file',
 550 #append => 'string',
 551 #initrd => 'file',
 552 ##soundhw => 'string',
 553
 554 while (my ($k, $v) = each %$confdesc) {
 555     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 556 }
 557
 558 my $MAX_IDE_DISKS = 4;
 559 my $MAX_SCSI_DISKS = 14;
 560 my $MAX_VIRTIO_DISKS = 16;
 561 my $MAX_SATA_DISKS = 6;
 562 my $MAX_USB_DEVICES = 5;
 563 my $MAX_NETS = 32;
 564 my $MAX_UNUSED_DISKS = 8;
 565 my $MAX_HOSTPCI_DEVICES = 4;
 566 my $MAX_SERIAL_PORTS = 4;
 567 my $MAX_PARALLEL_PORTS = 3;
 568 my $MAX_NUMA = 8;
 569
 570 my $numa_fmt = {
 571     cpus => {
 572         type => "string",
 573         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 574         description => "CPUs accessing this NUMA node.",
 575         format_description => "id[-id];...",
 576     },
 577     memory => {
 578         type => "number",
 579         description => "Amount of memory this NUMA node provides.",
 580         optional => 1,
 581     },
 582     hostnodes => {
 583         type => "string",
 584         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 585         description => "Host NUMA nodes to use.",
 586         format_description => "id[-id];...",
 587         optional => 1,
 588     },
 589     policy => {
 590         type => 'string',
 591         enum => [qw(preferred bind interleave)],
 592         description => "NUMA allocation policy.",
 593         optional => 1,
 594     },
 595 };
 596 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 597 my $numadesc = {
 598     optional => 1,
 599     type => 'string', format => $numa_fmt,
 600     description => "NUMA topology.",
 601 };
 602 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 603
 604 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 605     $confdesc->{"numa$i"} = $numadesc;
 606 }
 607
 608 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 609                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 610                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 611 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 612
 613 my $net_fmt_bridge_descr = <<__EOD__;
 614 Bridge to attach the network device to. The Proxmox VE standard bridge
 615 is called 'vmbr0'.
 616
 617 If you do not specify a bridge, we create a kvm user (NATed) network
 618 device, which provides DHCP and DNS services. The following addresses
 619 are used:
 620
 621  10.0.2.2   Gateway
 622  10.0.2.3   DNS Server
 623  10.0.2.4   SMB Server
 624
 625 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 626 __EOD__
 627
 628 my $net_fmt = {
 629     macaddr => {
 630         type => 'string',
 631         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 632         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 633         format_description => "XX:XX:XX:XX:XX:XX",
 634         optional => 1,
 635     },
 636     model => {
 637         type => 'string',
 638         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 639         enum => $nic_model_list,
 640         default_key => 1,
 641     },
 642     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 643     bridge => {
 644         type => 'string',
 645         description => $net_fmt_bridge_descr,
 646         format_description => 'bridge',
 647         optional => 1,
 648     },
 649     queues => {
 650         type => 'integer',
 651         minimum => 0, maximum => 16,
 652         description => 'Number of packet queues to be used on the device.',
 653         optional => 1,
 654     },
 655     rate => {
 656         type => 'number',
 657         minimum => 0,
 658         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 659         optional => 1,
 660     },
 661     tag => {
 662         type => 'integer',
 663         minimum => 1, maximum => 4094,
 664         description => 'VLAN tag to apply to packets on this interface.',
 665         optional => 1,
 666     },
 667     trunks => {
 668         type => 'string',
 669         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 670         description => 'VLAN trunks to pass through this interface.',
 671         format_description => 'vlanid[;vlanid...]',
 672         optional => 1,
 673     },
 674     firewall => {
 675         type => 'boolean',
 676         description => 'Whether this interface should be protected by the firewall.',
 677         optional => 1,
 678     },
 679     link_down => {
 680         type => 'boolean',
 681         description => 'Whether this interface should be disconnected (like pulling the plug).',
 682         optional => 1,
 683     },
 684 };
 685
 686 my $netdesc = {
 687     optional => 1,
 688     type => 'string', format => $net_fmt,
 689     description => "Specify network devices.",
 690 };
 691
 692 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 693
 694 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 695     $confdesc->{"net$i"} = $netdesc;
 696 }
 697
 698 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 699 sub verify_volume_id_or_qm_path {
 700     my ($volid, $noerr) = @_;
 701
 702     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 703         return $volid;
 704     }
 705
 706     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 707     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 708     if ($@) {
 709         return undef if $noerr;
 710         die $@;
 711     }
 712     return $volid;
 713 }
 714
 715 my $drivename_hash;
 716
 717 my %drivedesc_base = (
 718     volume => { alias => 'file' },
 719     file => {
 720         type => 'string',
 721         format => 'pve-volume-id-or-qm-path',
 722         default_key => 1,
 723         format_description => 'volume',
 724         description => "The drive's backing volume.",
 725     },
 726     media => {
 727         type => 'string',
 728         enum => [qw(cdrom disk)],
 729         description => "The drive's media type.",
 730         default => 'disk',
 731         optional => 1
 732     },
 733     cyls => {
 734         type => 'integer',
 735         description => "Force the drive's physical geometry to have a specific cylinder count.",
 736         optional => 1
 737     },
 738     heads => {
 739         type => 'integer',
 740         description => "Force the drive's physical geometry to have a specific head count.",
 741         optional => 1
 742     },
 743     secs => {
 744         type => 'integer',
 745         description => "Force the drive's physical geometry to have a specific sector count.",
 746         optional => 1
 747     },
 748     trans => {
 749         type => 'string',
 750         enum => [qw(none lba auto)],
 751         description => "Force disk geometry bios translation mode.",
 752         optional => 1,
 753     },
 754     snapshot => {
 755         type => 'boolean',
 756         description => "Whether the drive should be included when making snapshots.",
 757         optional => 1,
 758     },
 759     cache => {
 760         type => 'string',
 761         enum => [qw(none writethrough writeback unsafe directsync)],
 762         description => "The drive's cache mode",
 763         optional => 1,
 764     },
 765     format => get_standard_option('pve-qm-image-format'),
 766     size => {
 767         type => 'string',
 768         format => 'disk-size',
 769         format_description => 'DiskSize',
 770         description => "Disk size. This is purely informational and has no effect.",
 771         optional => 1,
 772     },
 773     backup => {
 774         type => 'boolean',
 775         description => "Whether the drive should be included when making backups.",
 776         optional => 1,
 777     },
 778     replicate => {
 779         type => 'boolean',
 780         description => 'Whether the drive should considered for replication jobs.',
 781         optional => 1,
 782         default => 1,
 783     },
 784     rerror => {
 785         type => 'string',
 786         enum => [qw(ignore report stop)],
 787         description => 'Read error action.',
 788         optional => 1,
 789     },
 790     werror => {
 791         type => 'string',
 792         enum => [qw(enospc ignore report stop)],
 793         description => 'Write error action.',
 794         optional => 1,
 795     },
 796     aio => {
 797         type => 'string',
 798         enum => [qw(native threads)],
 799         description => 'AIO type to use.',
 800         optional => 1,
 801     },
 802     discard => {
 803         type => 'string',
 804         enum => [qw(ignore on)],
 805         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 806         optional => 1,
 807     },
 808     detect_zeroes => {
 809         type => 'boolean',
 810         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 811         optional => 1,
 812     },
 813     serial => {
 814         type => 'string',
 815         format => 'urlencoded',
 816         format_description => 'serial',
 817         maxLength => 20*3, # *3 since it's %xx url enoded
 818         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 819         optional => 1,
 820     }
 821 );
 822
 823 my %iothread_fmt = ( iothread => {
 824         type => 'boolean',
 825         description => "Whether to use iothreads for this drive",
 826         optional => 1,
 827 });
 828
 829 my %model_fmt = (
 830     model => {
 831         type => 'string',
 832         format => 'urlencoded',
 833         format_description => 'model',
 834         maxLength => 40*3, # *3 since it's %xx url enoded
 835         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 836         optional => 1,
 837     },
 838 );
 839
 840 my %queues_fmt = (
 841     queues => {
 842         type => 'integer',
 843         description => "Number of queues.",
 844         minimum => 2,
 845         optional => 1
 846     }
 847 );
 848
 849 my %scsiblock_fmt = (
 850     scsiblock => {
 851         type => 'boolean',
 852         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 853         optional => 1,
 854         default => 0,
 855     },
 856 );
 857
 858 my $add_throttle_desc = sub {
 859     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 860     my $d = {
 861         type => $type,
 862         format_description => $unit,
 863         description => "Maximum $what in $longunit.",
 864         optional => 1,
 865     };
 866     $d->{minimum} = $minimum if defined($minimum);
 867     $drivedesc_base{$key} = $d;
 868 };
 869 # throughput: (leaky bucket)
 870 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 871 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 872 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 873 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 874 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 875 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 876 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 877 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 878 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 879
 880 # pools: (pool of IO before throttling starts taking effect)
 881 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 882 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 883 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 884 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 885 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 886 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 887
 888 # burst lengths
 889 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 890 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 891 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 892 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 893 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 894 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 895
 896 # legacy support
 897 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
 898 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
 899 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
 900 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
 901
 902 my $ide_fmt = {
 903     %drivedesc_base,
 904     %model_fmt,
 905 };
 906 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 907
 908 my $idedesc = {
 909     optional => 1,
 910     type => 'string', format => $ide_fmt,
 911     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 912 };
 913 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 914
 915 my $scsi_fmt = {
 916     %drivedesc_base,
 917     %iothread_fmt,
 918     %queues_fmt,
 919     %scsiblock_fmt,
 920 };
 921 my $scsidesc = {
 922     optional => 1,
 923     type => 'string', format => $scsi_fmt,
 924     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 925 };
 926 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 927
 928 my $sata_fmt = {
 929     %drivedesc_base,
 930 };
 931 my $satadesc = {
 932     optional => 1,
 933     type => 'string', format => $sata_fmt,
 934     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 935 };
 936 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 937
 938 my $virtio_fmt = {
 939     %drivedesc_base,
 940     %iothread_fmt,
 941 };
 942 my $virtiodesc = {
 943     optional => 1,
 944     type => 'string', format => $virtio_fmt,
 945     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 946 };
 947 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 948
 949 my $alldrive_fmt = {
 950     %drivedesc_base,
 951     %iothread_fmt,
 952     %model_fmt,
 953     %queues_fmt,
 954     %scsiblock_fmt,
 955 };
 956
 957 my $efidisk_fmt = {
 958     volume => { alias => 'file' },
 959     file => {
 960         type => 'string',
 961         format => 'pve-volume-id-or-qm-path',
 962         default_key => 1,
 963         format_description => 'volume',
 964         description => "The drive's backing volume.",
 965     },
 966     format => get_standard_option('pve-qm-image-format'),
 967     size => {
 968         type => 'string',
 969         format => 'disk-size',
 970         format_description => 'DiskSize',
 971         description => "Disk size. This is purely informational and has no effect.",
 972         optional => 1,
 973     },
 974 };
 975
 976 my $efidisk_desc = {
 977     optional => 1,
 978     type => 'string', format => $efidisk_fmt,
 979     description => "Configure a Disk for storing EFI vars",
 980 };
 981
 982 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 983
 984 my $usb_fmt = {
 985     host => {
 986         default_key => 1,
 987         type => 'string', format => 'pve-qm-usb-device',
 988         format_description => 'HOSTUSBDEVICE|spice',
 989         description => <<EODESCR,
 990 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 991
 992  'bus-port(.port)*' (decimal numbers) or
 993  'vendor_id:product_id' (hexadeciaml numbers) or
 994  'spice'
 995
 996 You can use the 'lsusb -t' command to list existing usb devices.
 997
 998 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 999
1000 The value 'spice' can be used to add a usb redirection devices for spice.
1001 EODESCR
1002     },
1003     usb3 => {
1004         optional => 1,
1005         type => 'boolean',
1006         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1007         default => 0,
1008     },
1009 };
1010
1011 my $usbdesc = {
1012     optional => 1,
1013     type => 'string', format => $usb_fmt,
1014     description => "Configure an USB device (n is 0 to 4).",
1015 };
1016 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1017
1018 # NOTE: the match-groups of this regex are used in parse_hostpci
1019 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1020 my $hostpci_fmt = {
1021     host => {
1022         default_key => 1,
1023         type => 'string',
1024         pattern => qr/$PCIRE(;$PCIRE)*/,
1025         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1026         description => <<EODESCR,
1027 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1028 of PCI virtual functions of the host. HOSTPCIID syntax is:
1029
1030 'bus:dev.func' (hexadecimal numbers)
1031
1032 You can us the 'lspci' command to list existing PCI devices.
1033 EODESCR
1034     },
1035     rombar => {
1036         type => 'boolean',
1037         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1038         optional => 1,
1039         default => 1,
1040     },
1041     romfile => {
1042         type => 'string',
1043         pattern => '[^,;]+',
1044         format_description => 'string',
1045         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1046         optional => 1,
1047     },
1048     pcie => {
1049         type => 'boolean',
1050         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1051         optional => 1,
1052         default => 0,
1053     },
1054     'x-vga' => {
1055         type => 'boolean',
1056         description =>  "Enable vfio-vga device support.",
1057         optional => 1,
1058         default => 0,
1059     },
1060 };
1061 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1062
1063 my $hostpcidesc = {
1064         optional => 1,
1065         type => 'string', format => 'pve-qm-hostpci',
1066         description => "Map host PCI devices into guest.",
1067         verbose_description =>  <<EODESCR,
1068 Map host PCI devices into guest.
1069
1070 NOTE: This option allows direct access to host hardware. So it is no longer
1071 possible to migrate such machines - use with special care.
1072
1073 CAUTION: Experimental! User reported problems with this option.
1074 EODESCR
1075 };
1076 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1077
1078 my $serialdesc = {
1079         optional => 1,
1080         type => 'string',
1081         pattern => '(/dev/.+|socket)',
1082         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1083         verbose_description =>  <<EODESCR,
1084 Create a serial device inside the VM (n is 0 to 3), and pass through a
1085 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1086 host side (use 'qm terminal' to open a terminal connection).
1087
1088 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1089
1090 CAUTION: Experimental! User reported problems with this option.
1091 EODESCR
1092 };
1093
1094 my $paralleldesc= {
1095         optional => 1,
1096         type => 'string',
1097         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1098         description =>  "Map host parallel devices (n is 0 to 2).",
1099         verbose_description =>  <<EODESCR,
1100 Map host parallel devices (n is 0 to 2).
1101
1102 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1103
1104 CAUTION: Experimental! User reported problems with this option.
1105 EODESCR
1106 };
1107
1108 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1109     $confdesc->{"parallel$i"} = $paralleldesc;
1110 }
1111
1112 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1113     $confdesc->{"serial$i"} = $serialdesc;
1114 }
1115
1116 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1117     $confdesc->{"hostpci$i"} = $hostpcidesc;
1118 }
1119
1120 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1121     $drivename_hash->{"ide$i"} = 1;
1122     $confdesc->{"ide$i"} = $idedesc;
1123 }
1124
1125 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1126     $drivename_hash->{"sata$i"} = 1;
1127     $confdesc->{"sata$i"} = $satadesc;
1128 }
1129
1130 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1131     $drivename_hash->{"scsi$i"} = 1;
1132     $confdesc->{"scsi$i"} = $scsidesc ;
1133 }
1134
1135 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1136     $drivename_hash->{"virtio$i"} = 1;
1137     $confdesc->{"virtio$i"} = $virtiodesc;
1138 }
1139
1140 $drivename_hash->{efidisk0} = 1;
1141 $confdesc->{efidisk0} = $efidisk_desc;
1142
1143 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1144     $confdesc->{"usb$i"} = $usbdesc;
1145 }
1146
1147 my $unuseddesc = {
1148     optional => 1,
1149     type => 'string', format => 'pve-volume-id',
1150     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1151 };
1152
1153 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1154     $confdesc->{"unused$i"} = $unuseddesc;
1155 }
1156
1157 my $kvm_api_version = 0;
1158
1159 sub kvm_version {
1160
1161     return $kvm_api_version if $kvm_api_version;
1162
1163     my $fh = IO::File->new("</dev/kvm") ||
1164         return 0;
1165
1166     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1167         $kvm_api_version = $v;
1168     }
1169
1170     $fh->close();
1171
1172     return  $kvm_api_version;
1173 }
1174
1175 my $kvm_user_version;
1176
1177 sub kvm_user_version {
1178
1179     return $kvm_user_version if $kvm_user_version;
1180
1181     $kvm_user_version = 'unknown';
1182
1183     my $code = sub {
1184         my $line = shift;
1185         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1186             $kvm_user_version = $2;
1187         }
1188     };
1189
1190     eval { run_command("kvm -version", outfunc => $code); };
1191     warn $@ if $@;
1192
1193     return $kvm_user_version;
1194
1195 }
1196
1197 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1198
1199 sub valid_drive_names {
1200     # order is important - used to autoselect boot disk
1201     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1202             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1203             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1204             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1205             'efidisk0');
1206 }
1207
1208 sub is_valid_drivename {
1209     my $dev = shift;
1210
1211     return defined($drivename_hash->{$dev});
1212 }
1213
1214 sub option_exists {
1215     my $key = shift;
1216     return defined($confdesc->{$key});
1217 }
1218
1219 sub nic_models {
1220     return $nic_model_list;
1221 }
1222
1223 sub os_list_description {
1224
1225     return {
1226         other => 'Other',
1227         wxp => 'Windows XP',
1228         w2k => 'Windows 2000',
1229         w2k3 =>, 'Windows 2003',
1230         w2k8 => 'Windows 2008',
1231         wvista => 'Windows Vista',
1232         win7 => 'Windows 7',
1233         win8 => 'Windows 8/2012',
1234         win10 => 'Windows 10/2016',
1235         l24 => 'Linux 2.4',
1236         l26 => 'Linux 2.6',
1237     };
1238 }
1239
1240 my $cdrom_path;
1241
1242 sub get_cdrom_path {
1243
1244     return  $cdrom_path if $cdrom_path;
1245
1246     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1247     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1248     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1249 }
1250
1251 sub get_iso_path {
1252     my ($storecfg, $vmid, $cdrom) = @_;
1253
1254     if ($cdrom eq 'cdrom') {
1255         return get_cdrom_path();
1256     } elsif ($cdrom eq 'none') {
1257         return '';
1258     } elsif ($cdrom =~ m|^/|) {
1259         return $cdrom;
1260     } else {
1261         return PVE::Storage::path($storecfg, $cdrom);
1262     }
1263 }
1264
1265 # try to convert old style file names to volume IDs
1266 sub filename_to_volume_id {
1267     my ($vmid, $file, $media) = @_;
1268
1269     if (!($file eq 'none' || $file eq 'cdrom' ||
1270           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1271
1272         return undef if $file =~ m|/|;
1273
1274         if ($media && $media eq 'cdrom') {
1275             $file = "local:iso/$file";
1276         } else {
1277             $file = "local:$vmid/$file";
1278         }
1279     }
1280
1281     return $file;
1282 }
1283
1284 sub verify_media_type {
1285     my ($opt, $vtype, $media) = @_;
1286
1287     return if !$media;
1288
1289     my $etype;
1290     if ($media eq 'disk') {
1291         $etype = 'images';
1292     } elsif ($media eq 'cdrom') {
1293         $etype = 'iso';
1294     } else {
1295         die "internal error";
1296     }
1297
1298     return if ($vtype eq $etype);
1299
1300     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1301 }
1302
1303 sub cleanup_drive_path {
1304     my ($opt, $storecfg, $drive) = @_;
1305
1306     # try to convert filesystem paths to volume IDs
1307
1308     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1309         ($drive->{file} !~ m|^/dev/.+|) &&
1310         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1311         ($drive->{file} !~ m/^\d+$/)) {
1312         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1313         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1314         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1315         verify_media_type($opt, $vtype, $drive->{media});
1316         $drive->{file} = $volid;
1317     }
1318
1319     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1320 }
1321
1322 sub parse_hotplug_features {
1323     my ($data) = @_;
1324
1325     my $res = {};
1326
1327     return $res if $data eq '0';
1328
1329     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1330
1331     foreach my $feature (PVE::Tools::split_list($data)) {
1332         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1333             $res->{$1} = 1;
1334         } else {
1335             die "invalid hotplug feature '$feature'\n";
1336         }
1337     }
1338     return $res;
1339 }
1340
1341 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1342 sub pve_verify_hotplug_features {
1343     my ($value, $noerr) = @_;
1344
1345     return $value if parse_hotplug_features($value);
1346
1347     return undef if $noerr;
1348
1349     die "unable to parse hotplug option\n";
1350 }
1351
1352 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1353 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1354 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1355 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1356 #        [,iothread=on][,serial=serial][,model=model]
1357
1358 sub parse_drive {
1359     my ($key, $data) = @_;
1360
1361     my ($interface, $index);
1362
1363     if ($key =~ m/^([^\d]+)(\d+)$/) {
1364         $interface = $1;
1365         $index = $2;
1366     } else {
1367         return undef;
1368     }
1369
1370     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1371                                      : $confdesc->{$key}->{format};
1372     if (!$desc) {
1373         warn "invalid drive key: $key\n";
1374         return undef;
1375     }
1376     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1377     return undef if !$res;
1378     $res->{interface} = $interface;
1379     $res->{index} = $index;
1380
1381     my $error = 0;
1382     foreach my $opt (qw(bps bps_rd bps_wr)) {
1383         if (my $bps = defined(delete $res->{$opt})) {
1384             if (defined($res->{"m$opt"})) {
1385                 warn "both $opt and m$opt specified\n";
1386                 ++$error;
1387                 next;
1388             }
1389             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1390         }
1391     }
1392
1393     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1394     for my $requirement (
1395         [mbps_max => 'mbps'],
1396         [mbps_rd_max => 'mbps_rd'],
1397         [mbps_wr_max => 'mbps_wr'],
1398         [miops_max => 'miops'],
1399         [miops_rd_max => 'miops_rd'],
1400         [miops_wr_max => 'miops_wr'],
1401         [bps_max_length => 'mbps_max'],
1402         [bps_rd_max_length => 'mbps_rd_max'],
1403         [bps_wr_max_length => 'mbps_wr_max'],
1404         [iops_max_length => 'iops_max'],
1405         [iops_rd_max_length => 'iops_rd_max'],
1406         [iops_wr_max_length => 'iops_wr_max']) {
1407         my ($option, $requires) = @$requirement;
1408         if ($res->{$option} && !$res->{$requires}) {
1409             warn "$option requires $requires\n";
1410             ++$error;
1411         }
1412     }
1413
1414     return undef if $error;
1415
1416     return undef if $res->{mbps_rd} && $res->{mbps};
1417     return undef if $res->{mbps_wr} && $res->{mbps};
1418     return undef if $res->{iops_rd} && $res->{iops};
1419     return undef if $res->{iops_wr} && $res->{iops};
1420
1421     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1422         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1423         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1424         return undef if $res->{interface} eq 'virtio';
1425     }
1426
1427     if (my $size = $res->{size}) {
1428         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1429     }
1430
1431     return $res;
1432 }
1433
1434 sub print_drive {
1435     my ($vmid, $drive) = @_;
1436     my $data = { %$drive };
1437     delete $data->{$_} for qw(index interface);
1438     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1439 }
1440
1441 sub scsi_inquiry {
1442     my($fh, $noerr) = @_;
1443
1444     my $SG_IO = 0x2285;
1445     my $SG_GET_VERSION_NUM = 0x2282;
1446
1447     my $versionbuf = "\x00" x 8;
1448     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1449     if (!$ret) {
1450         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1451         return undef;
1452     }
1453     my $version = unpack("I", $versionbuf);
1454     if ($version < 30000) {
1455         die "scsi generic interface too old\n"  if !$noerr;
1456         return undef;
1457     }
1458
1459     my $buf = "\x00" x 36;
1460     my $sensebuf = "\x00" x 8;
1461     my $cmd = pack("C x3 C x1", 0x12, 36);
1462
1463     # see /usr/include/scsi/sg.h
1464     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1465
1466     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1467                       length($sensebuf), 0, length($buf), $buf,
1468                       $cmd, $sensebuf, 6000);
1469
1470     $ret = ioctl($fh, $SG_IO, $packet);
1471     if (!$ret) {
1472         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1473         return undef;
1474     }
1475
1476     my @res = unpack($sg_io_hdr_t, $packet);
1477     if ($res[17] || $res[18]) {
1478         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1479         return undef;
1480     }
1481
1482     my $res = {};
1483     (my $byte0, my $byte1, $res->{vendor},
1484      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1485
1486     $res->{removable} = $byte1 & 128 ? 1 : 0;
1487     $res->{type} = $byte0 & 31;
1488
1489     return $res;
1490 }
1491
1492 sub path_is_scsi {
1493     my ($path) = @_;
1494
1495     my $fh = IO::File->new("+<$path") || return undef;
1496     my $res = scsi_inquiry($fh, 1);
1497     close($fh);
1498
1499     return $res;
1500 }
1501
1502 sub machine_type_is_q35 {
1503     my ($conf) = @_;
1504
1505     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1506 }
1507
1508 sub print_tabletdevice_full {
1509     my ($conf) = @_;
1510
1511     my $q35 = machine_type_is_q35($conf);
1512
1513     # we use uhci for old VMs because tablet driver was buggy in older qemu
1514     my $usbbus = $q35 ? "ehci" : "uhci";
1515
1516     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1517 }
1518
1519 sub print_drivedevice_full {
1520     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1521
1522     my $device = '';
1523     my $maxdev = 0;
1524
1525     if ($drive->{interface} eq 'virtio') {
1526         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1527         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1528         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1529     } elsif ($drive->{interface} eq 'scsi') {
1530
1531         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1532         my $unit = $drive->{index} % $maxdev;
1533         my $devicetype = 'hd';
1534         my $path = '';
1535         if (drive_is_cdrom($drive)) {
1536             $devicetype = 'cd';
1537         } else {
1538             if ($drive->{file} =~ m|^/|) {
1539                 $path = $drive->{file};
1540                 if (my $info = path_is_scsi($path)) {
1541                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1542                         $devicetype = 'block';
1543                     } elsif ($info->{type} == 1) { # tape
1544                         $devicetype = 'generic';
1545                     }
1546                 }
1547             } else {
1548                  $path = PVE::Storage::path($storecfg, $drive->{file});
1549             }
1550
1551             if($path =~ m/^iscsi\:\/\//){
1552                 $devicetype = 'generic';
1553             }
1554         }
1555
1556         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1557             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1558         } else {
1559             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1560         }
1561
1562     } elsif ($drive->{interface} eq 'ide'){
1563         $maxdev = 2;
1564         my $controller = int($drive->{index} / $maxdev);
1565         my $unit = $drive->{index} % $maxdev;
1566         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1567
1568         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1569         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1570             $model = URI::Escape::uri_unescape($model);
1571             $device .= ",model=$model";
1572         }
1573     } elsif ($drive->{interface} eq 'sata'){
1574         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1575         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1576         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1577     } elsif ($drive->{interface} eq 'usb') {
1578         die "implement me";
1579         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1580     } else {
1581         die "unsupported interface type";
1582     }
1583
1584     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1585
1586     return $device;
1587 }
1588
1589 sub get_initiator_name {
1590     my $initiator;
1591
1592     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1593     while (defined(my $line = <$fh>)) {
1594         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1595         $initiator = $1;
1596         last;
1597     }
1598     $fh->close();
1599
1600     return $initiator;
1601 }
1602
1603 sub print_drive_full {
1604     my ($storecfg, $vmid, $drive) = @_;
1605
1606     my $path;
1607     my $volid = $drive->{file};
1608     my $format;
1609
1610     if (drive_is_cdrom($drive)) {
1611         $path = get_iso_path($storecfg, $vmid, $volid);
1612     } else {
1613         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1614         if ($storeid) {
1615             $path = PVE::Storage::path($storecfg, $volid);
1616             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1617             $format = qemu_img_format($scfg, $volname);
1618         } else {
1619             $path = $volid;
1620             $format = "raw";
1621         }
1622    }
1623
1624     my $opts = '';
1625     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard);
1626     foreach my $o (@qemu_drive_options) {
1627         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1628     }
1629     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1630         my ($dir, $qmpname) = @$type;
1631         if (my $v = $drive->{"mbps$dir"}) {
1632             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1633         }
1634         if (my $v = $drive->{"mbps${dir}_max"}) {
1635             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1636         }
1637         if (my $v = $drive->{"bps${dir}_max_length"}) {
1638             $opts .= ",throttling.bps$qmpname-max-length=$v";
1639         }
1640         if (my $v = $drive->{"iops${dir}"}) {
1641             $opts .= ",throttling.iops$qmpname=$v";
1642         }
1643         if (my $v = $drive->{"iops${dir}_max"}) {
1644             $opts .= ",throttling.iops$qmpname-max=$v";
1645         }
1646         if (my $v = $drive->{"iops${dir}_max_length"}) {
1647             $opts .= ",throttling.iops$qmpname-max-length=$v";
1648         }
1649     }
1650
1651     if (my $serial = $drive->{serial}) {
1652         $serial = URI::Escape::uri_unescape($serial);
1653         $opts .= ",serial=$serial";
1654     }
1655
1656     $opts .= ",format=$format" if $format && !$drive->{format};
1657
1658     my $cache_direct = 0;
1659
1660     if (my $cache = $drive->{cache}) {
1661         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1662     } elsif (!drive_is_cdrom($drive)) {
1663         $opts .= ",cache=none";
1664         $cache_direct = 1;
1665     }
1666
1667     # aio native works only with O_DIRECT
1668     if (!$drive->{aio}) {
1669         if($cache_direct) {
1670             $opts .= ",aio=native";
1671         } else {
1672             $opts .= ",aio=threads";
1673         }
1674     }
1675
1676     if (!drive_is_cdrom($drive)) {
1677         my $detectzeroes;
1678         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1679             $detectzeroes = 'off';
1680         } elsif ($drive->{discard}) {
1681             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1682         } else {
1683             # This used to be our default with discard not being specified:
1684             $detectzeroes = 'on';
1685         }
1686         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1687     }
1688
1689     my $pathinfo = $path ? "file=$path," : '';
1690
1691     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1692 }
1693
1694 sub print_netdevice_full {
1695     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1696
1697     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1698
1699     my $device = $net->{model};
1700     if ($net->{model} eq 'virtio') {
1701          $device = 'virtio-net-pci';
1702      };
1703
1704     my $pciaddr = print_pci_addr("$netid", $bridges);
1705     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1706     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1707         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1708         my $vectors = $net->{queues} * 2 + 2;
1709         $tmpstr .= ",vectors=$vectors,mq=on";
1710     }
1711     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1712
1713     if ($use_old_bios_files) {
1714         my $romfile;
1715         if ($device eq 'virtio-net-pci') {
1716             $romfile = 'pxe-virtio.rom';
1717         } elsif ($device eq 'e1000') {
1718             $romfile = 'pxe-e1000.rom';
1719         } elsif ($device eq 'ne2k') {
1720             $romfile = 'pxe-ne2k_pci.rom';
1721         } elsif ($device eq 'pcnet') {
1722             $romfile = 'pxe-pcnet.rom';
1723         } elsif ($device eq 'rtl8139') {
1724             $romfile = 'pxe-rtl8139.rom';
1725         }
1726         $tmpstr .= ",romfile=$romfile" if $romfile;
1727     }
1728
1729     return $tmpstr;
1730 }
1731
1732 sub print_netdev_full {
1733     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1734
1735     my $i = '';
1736     if ($netid =~ m/^net(\d+)$/) {
1737         $i = int($1);
1738     }
1739
1740     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1741
1742     my $ifname = "tap${vmid}i$i";
1743
1744     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1745     die "interface name '$ifname' is too long (max 15 character)\n"
1746         if length($ifname) >= 16;
1747
1748     my $vhostparam = '';
1749     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1750
1751     my $vmname = $conf->{name} || "vm$vmid";
1752
1753     my $netdev = "";
1754     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1755
1756     if ($net->{bridge}) {
1757         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1758     } else {
1759         $netdev = "type=user,id=$netid,hostname=$vmname";
1760     }
1761
1762     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1763
1764     return $netdev;
1765 }
1766
1767
1768 sub print_cpu_device {
1769     my ($conf, $id) = @_;
1770
1771     my $kvm = $conf->{kvm} // 1;
1772     my $cpu = $kvm ? "kvm64" : "qemu64";
1773     if (my $cputype = $conf->{cpu}) {
1774         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1775             or die "Cannot parse cpu description: $cputype\n";
1776         $cpu = $cpuconf->{cputype};
1777     }
1778
1779     my $cores = $conf->{cores} || 1;
1780
1781     my $current_core = ($id - 1) % $cores;
1782     my $current_socket = int(($id - 1 - $current_core)/$cores);
1783
1784     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1785 }
1786
1787 sub drive_is_cdrom {
1788     my ($drive) = @_;
1789
1790     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1791
1792 }
1793
1794 sub parse_number_sets {
1795     my ($set) = @_;
1796     my $res = [];
1797     foreach my $part (split(/;/, $set)) {
1798         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1799             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1800             push @$res, [ $1, $2 ];
1801         } else {
1802             die "invalid range: $part\n";
1803         }
1804     }
1805     return $res;
1806 }
1807
1808 sub parse_numa {
1809     my ($data) = @_;
1810
1811     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1812     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1813     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1814     return $res;
1815 }
1816
1817 sub parse_hostpci {
1818     my ($value) = @_;
1819
1820     return undef if !$value;
1821
1822     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1823
1824     my @idlist = split(/;/, $res->{host});
1825     delete $res->{host};
1826     foreach my $id (@idlist) {
1827         if ($id =~ /^$PCIRE$/) {
1828             if (defined($2)) {
1829                 push @{$res->{pciid}}, { id => $1, function => $2 };
1830             } else {
1831                 my $pcidevices = lspci($1);
1832                 $res->{pciid} = $pcidevices->{$1};
1833             }
1834         } else {
1835             # should have been caught by parse_property_string already
1836             die "failed to parse PCI id: $id\n";
1837         }
1838     }
1839     return $res;
1840 }
1841
1842 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1843 sub parse_net {
1844     my ($data) = @_;
1845
1846     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1847     if ($@) {
1848         warn $@;
1849         return undef;
1850     }
1851     if (!defined($res->{macaddr})) {
1852         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1853         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1854     }
1855     return $res;
1856 }
1857
1858 sub print_net {
1859     my $net = shift;
1860
1861     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1862 }
1863
1864 sub add_random_macs {
1865     my ($settings) = @_;
1866
1867     foreach my $opt (keys %$settings) {
1868         next if $opt !~ m/^net(\d+)$/;
1869         my $net = parse_net($settings->{$opt});
1870         next if !$net;
1871         $settings->{$opt} = print_net($net);
1872     }
1873 }
1874
1875 sub vm_is_volid_owner {
1876     my ($storecfg, $vmid, $volid) = @_;
1877
1878     if ($volid !~  m|^/|) {
1879         my ($path, $owner);
1880         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1881         if ($owner && ($owner == $vmid)) {
1882             return 1;
1883         }
1884     }
1885
1886     return undef;
1887 }
1888
1889 sub split_flagged_list {
1890     my $text = shift || '';
1891     $text =~ s/[,;]/ /g;
1892     $text =~ s/^\s+//;
1893     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1894 }
1895
1896 sub join_flagged_list {
1897     my ($how, $lst) = @_;
1898     join $how, map { $lst->{$_} . $_ } keys %$lst;
1899 }
1900
1901 sub vmconfig_delete_pending_option {
1902     my ($conf, $key, $force) = @_;
1903
1904     delete $conf->{pending}->{$key};
1905     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1906     $pending_delete_hash->{$key} = $force ? '!' : '';
1907     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1908 }
1909
1910 sub vmconfig_undelete_pending_option {
1911     my ($conf, $key) = @_;
1912
1913     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1914     delete $pending_delete_hash->{$key};
1915
1916     if (%$pending_delete_hash) {
1917         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1918     } else {
1919         delete $conf->{pending}->{delete};
1920     }
1921 }
1922
1923 sub vmconfig_register_unused_drive {
1924     my ($storecfg, $vmid, $conf, $drive) = @_;
1925
1926     if (!drive_is_cdrom($drive)) {
1927         my $volid = $drive->{file};
1928         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1929             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1930         }
1931     }
1932 }
1933
1934 sub vmconfig_cleanup_pending {
1935     my ($conf) = @_;
1936
1937     # remove pending changes when nothing changed
1938     my $changes;
1939     foreach my $opt (keys %{$conf->{pending}}) {
1940         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1941             $changes = 1;
1942             delete $conf->{pending}->{$opt};
1943         }
1944     }
1945
1946     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1947     my $pending_delete_hash = {};
1948     while (my ($opt, $force) = each %$current_delete_hash) {
1949         if (defined($conf->{$opt})) {
1950             $pending_delete_hash->{$opt} = $force;
1951         } else {
1952             $changes = 1;
1953         }
1954     }
1955
1956     if (%$pending_delete_hash) {
1957         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1958     } else {
1959         delete $conf->{pending}->{delete};
1960     }
1961
1962     return $changes;
1963 }
1964
1965 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1966 my $smbios1_fmt = {
1967     uuid => {
1968         type => 'string',
1969         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1970         format_description => 'UUID',
1971         description => "Set SMBIOS1 UUID.",
1972         optional => 1,
1973     },
1974     version => {
1975         type => 'string',
1976         pattern => '\S+',
1977         format_description => 'string',
1978         description => "Set SMBIOS1 version.",
1979         optional => 1,
1980     },
1981     serial => {
1982         type => 'string',
1983         pattern => '\S+',
1984         format_description => 'string',
1985         description => "Set SMBIOS1 serial number.",
1986         optional => 1,
1987     },
1988     manufacturer => {
1989         type => 'string',
1990         pattern => '\S+',
1991         format_description => 'string',
1992         description => "Set SMBIOS1 manufacturer.",
1993         optional => 1,
1994     },
1995     product => {
1996         type => 'string',
1997         pattern => '\S+',
1998         format_description => 'string',
1999         description => "Set SMBIOS1 product ID.",
2000         optional => 1,
2001     },
2002     sku => {
2003         type => 'string',
2004         pattern => '\S+',
2005         format_description => 'string',
2006         description => "Set SMBIOS1 SKU string.",
2007         optional => 1,
2008     },
2009     family => {
2010         type => 'string',
2011         pattern => '\S+',
2012         format_description => 'string',
2013         description => "Set SMBIOS1 family string.",
2014         optional => 1,
2015     },
2016 };
2017
2018 sub parse_smbios1 {
2019     my ($data) = @_;
2020
2021     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2022     warn $@ if $@;
2023     return $res;
2024 }
2025
2026 sub print_smbios1 {
2027     my ($smbios1) = @_;
2028     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2029 }
2030
2031 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2032
2033 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2034 sub verify_bootdisk {
2035     my ($value, $noerr) = @_;
2036
2037     return $value if is_valid_drivename($value);
2038
2039     return undef if $noerr;
2040
2041     die "invalid boot disk '$value'\n";
2042 }
2043
2044 sub parse_watchdog {
2045     my ($value) = @_;
2046
2047     return undef if !$value;
2048
2049     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2050     warn $@ if $@;
2051     return $res;
2052 }
2053
2054 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2055 sub verify_usb_device {
2056     my ($value, $noerr) = @_;
2057
2058     return $value if parse_usb_device($value);
2059
2060     return undef if $noerr;
2061
2062     die "unable to parse usb device\n";
2063 }
2064
2065 # add JSON properties for create and set function
2066 sub json_config_properties {
2067     my $prop = shift;
2068
2069     foreach my $opt (keys %$confdesc) {
2070         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2071         $prop->{$opt} = $confdesc->{$opt};
2072     }
2073
2074     return $prop;
2075 }
2076
2077 sub check_type {
2078     my ($key, $value) = @_;
2079
2080     die "unknown setting '$key'\n" if !$confdesc->{$key};
2081
2082     my $type = $confdesc->{$key}->{type};
2083
2084     if (!defined($value)) {
2085         die "got undefined value\n";
2086     }
2087
2088     if ($value =~ m/[\n\r]/) {
2089         die "property contains a line feed\n";
2090     }
2091
2092     if ($type eq 'boolean') {
2093         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2094         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2095         die "type check ('boolean') failed - got '$value'\n";
2096     } elsif ($type eq 'integer') {
2097         return int($1) if $value =~ m/^(\d+)$/;
2098         die "type check ('integer') failed - got '$value'\n";
2099     } elsif ($type eq 'number') {
2100         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2101         die "type check ('number') failed - got '$value'\n";
2102     } elsif ($type eq 'string') {
2103         if (my $fmt = $confdesc->{$key}->{format}) {
2104             PVE::JSONSchema::check_format($fmt, $value);
2105             return $value;
2106         }
2107         $value =~ s/^\"(.*)\"$/$1/;
2108         return $value;
2109     } else {
2110         die "internal error"
2111     }
2112 }
2113
2114 sub check_iommu_support{
2115     #fixme : need to check IOMMU support
2116     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2117
2118     my $iommu=1;
2119     return $iommu;
2120
2121 }
2122
2123 sub touch_config {
2124     my ($vmid) = @_;
2125
2126     my $conf = PVE::QemuConfig->config_file($vmid);
2127     utime undef, undef, $conf;
2128 }
2129
2130 sub destroy_vm {
2131     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2132
2133     my $conffile = PVE::QemuConfig->config_file($vmid);
2134
2135     my $conf = PVE::QemuConfig->load_config($vmid);
2136
2137     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2138
2139     if ($conf->{template}) {
2140         # check if any base image is still used by a linked clone
2141         foreach_drive($conf, sub {
2142                 my ($ds, $drive) = @_;
2143
2144                 return if drive_is_cdrom($drive);
2145
2146                 my $volid = $drive->{file};
2147
2148                 return if !$volid || $volid =~ m|^/|;
2149
2150                 die "base volume '$volid' is still in use by linked cloned\n"
2151                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2152
2153         });
2154     }
2155
2156     # only remove disks owned by this VM
2157     foreach_drive($conf, sub {
2158         my ($ds, $drive) = @_;
2159
2160         return if drive_is_cdrom($drive);
2161
2162         my $volid = $drive->{file};
2163
2164         return if !$volid || $volid =~ m|^/|;
2165
2166         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2167         return if !$path || !$owner || ($owner != $vmid);
2168
2169         eval {
2170             PVE::Storage::vdisk_free($storecfg, $volid);
2171         };
2172         warn "Could not remove disk '$volid', check manually: $@" if $@;
2173
2174     });
2175
2176     if ($keep_empty_config) {
2177         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2178     } else {
2179         unlink $conffile;
2180     }
2181
2182     # also remove unused disk
2183     eval {
2184         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2185
2186         eval {
2187             PVE::Storage::foreach_volid($dl, sub {
2188                 my ($volid, $sid, $volname, $d) = @_;
2189                 PVE::Storage::vdisk_free($storecfg, $volid);
2190             });
2191         };
2192         warn $@ if $@;
2193
2194     };
2195     warn $@ if $@;
2196 }
2197
2198 sub parse_vm_config {
2199     my ($filename, $raw) = @_;
2200
2201     return undef if !defined($raw);
2202
2203     my $res = {
2204         digest => Digest::SHA::sha1_hex($raw),
2205         snapshots => {},
2206         pending => {},
2207     };
2208
2209     $filename =~ m|/qemu-server/(\d+)\.conf$|
2210         || die "got strange filename '$filename'";
2211
2212     my $vmid = $1;
2213
2214     my $conf = $res;
2215     my $descr;
2216     my $section = '';
2217
2218     my @lines = split(/\n/, $raw);
2219     foreach my $line (@lines) {
2220         next if $line =~ m/^\s*$/;
2221
2222         if ($line =~ m/^\[PENDING\]\s*$/i) {
2223             $section = 'pending';
2224             if (defined($descr)) {
2225                 $descr =~ s/\s+$//;
2226                 $conf->{description} = $descr;
2227             }
2228             $descr = undef;
2229             $conf = $res->{$section} = {};
2230             next;
2231
2232         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2233             $section = $1;
2234             if (defined($descr)) {
2235                 $descr =~ s/\s+$//;
2236                 $conf->{description} = $descr;
2237             }
2238             $descr = undef;
2239             $conf = $res->{snapshots}->{$section} = {};
2240             next;
2241         }
2242
2243         if ($line =~ m/^\#(.*)\s*$/) {
2244             $descr = '' if !defined($descr);
2245             $descr .= PVE::Tools::decode_text($1) . "\n";
2246             next;
2247         }
2248
2249         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2250             $descr = '' if !defined($descr);
2251             $descr .= PVE::Tools::decode_text($2);
2252         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2253             $conf->{snapstate} = $1;
2254         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2255             my $key = $1;
2256             my $value = $2;
2257             $conf->{$key} = $value;
2258         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2259             my $value = $1;
2260             if ($section eq 'pending') {
2261                 $conf->{delete} = $value; # we parse this later
2262             } else {
2263                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2264             }
2265         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2266             my $key = $1;
2267             my $value = $2;
2268             eval { $value = check_type($key, $value); };
2269             if ($@) {
2270                 warn "vm $vmid - unable to parse value of '$key' - $@";
2271             } else {
2272                 $key = 'ide2' if $key eq 'cdrom';
2273                 my $fmt = $confdesc->{$key}->{format};
2274                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2275                     my $v = parse_drive($key, $value);
2276                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2277                         $v->{file} = $volid;
2278                         $value = print_drive($vmid, $v);
2279                     } else {
2280                         warn "vm $vmid - unable to parse value of '$key'\n";
2281                         next;
2282                     }
2283                 }
2284
2285                 $conf->{$key} = $value;
2286             }
2287         }
2288     }
2289
2290     if (defined($descr)) {
2291         $descr =~ s/\s+$//;
2292         $conf->{description} = $descr;
2293     }
2294     delete $res->{snapstate}; # just to be sure
2295
2296     return $res;
2297 }
2298
2299 sub write_vm_config {
2300     my ($filename, $conf) = @_;
2301
2302     delete $conf->{snapstate}; # just to be sure
2303
2304     if ($conf->{cdrom}) {
2305         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2306         $conf->{ide2} = $conf->{cdrom};
2307         delete $conf->{cdrom};
2308     }
2309
2310     # we do not use 'smp' any longer
2311     if ($conf->{sockets}) {
2312         delete $conf->{smp};
2313     } elsif ($conf->{smp}) {
2314         $conf->{sockets} = $conf->{smp};
2315         delete $conf->{cores};
2316         delete $conf->{smp};
2317     }
2318
2319     my $used_volids = {};
2320
2321     my $cleanup_config = sub {
2322         my ($cref, $pending, $snapname) = @_;
2323
2324         foreach my $key (keys %$cref) {
2325             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2326                 $key eq 'snapstate' || $key eq 'pending';
2327             my $value = $cref->{$key};
2328             if ($key eq 'delete') {
2329                 die "propertry 'delete' is only allowed in [PENDING]\n"
2330                     if !$pending;
2331                 # fixme: check syntax?
2332                 next;
2333             }
2334             eval { $value = check_type($key, $value); };
2335             die "unable to parse value of '$key' - $@" if $@;
2336
2337             $cref->{$key} = $value;
2338
2339             if (!$snapname && is_valid_drivename($key)) {
2340                 my $drive = parse_drive($key, $value);
2341                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2342             }
2343         }
2344     };
2345
2346     &$cleanup_config($conf);
2347
2348     &$cleanup_config($conf->{pending}, 1);
2349
2350     foreach my $snapname (keys %{$conf->{snapshots}}) {
2351         die "internal error" if $snapname eq 'pending';
2352         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2353     }
2354
2355     # remove 'unusedX' settings if we re-add a volume
2356     foreach my $key (keys %$conf) {
2357         my $value = $conf->{$key};
2358         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2359             delete $conf->{$key};
2360         }
2361     }
2362
2363     my $generate_raw_config = sub {
2364         my ($conf, $pending) = @_;
2365
2366         my $raw = '';
2367
2368         # add description as comment to top of file
2369         if (defined(my $descr = $conf->{description})) {
2370             if ($descr) {
2371                 foreach my $cl (split(/\n/, $descr)) {
2372                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2373                 }
2374             } else {
2375                 $raw .= "#\n" if $pending;
2376             }
2377         }
2378
2379         foreach my $key (sort keys %$conf) {
2380             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2381             $raw .= "$key: $conf->{$key}\n";
2382         }
2383         return $raw;
2384     };
2385
2386     my $raw = &$generate_raw_config($conf);
2387
2388     if (scalar(keys %{$conf->{pending}})){
2389         $raw .= "\n[PENDING]\n";
2390         $raw .= &$generate_raw_config($conf->{pending}, 1);
2391     }
2392
2393     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2394         $raw .= "\n[$snapname]\n";
2395         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2396     }
2397
2398     return $raw;
2399 }
2400
2401 sub load_defaults {
2402
2403     my $res = {};
2404
2405     # we use static defaults from our JSON schema configuration
2406     foreach my $key (keys %$confdesc) {
2407         if (defined(my $default = $confdesc->{$key}->{default})) {
2408             $res->{$key} = $default;
2409         }
2410     }
2411
2412     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2413     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2414
2415     return $res;
2416 }
2417
2418 sub config_list {
2419     my $vmlist = PVE::Cluster::get_vmlist();
2420     my $res = {};
2421     return $res if !$vmlist || !$vmlist->{ids};
2422     my $ids = $vmlist->{ids};
2423
2424     foreach my $vmid (keys %$ids) {
2425         my $d = $ids->{$vmid};
2426         next if !$d->{node} || $d->{node} ne $nodename;
2427         next if !$d->{type} || $d->{type} ne 'qemu';
2428         $res->{$vmid}->{exists} = 1;
2429     }
2430     return $res;
2431 }
2432
2433 # test if VM uses local resources (to prevent migration)
2434 sub check_local_resources {
2435     my ($conf, $noerr) = @_;
2436
2437     my $loc_res = 0;
2438
2439     $loc_res = 1 if $conf->{hostusb}; # old syntax
2440     $loc_res = 1 if $conf->{hostpci}; # old syntax
2441
2442     foreach my $k (keys %$conf) {
2443         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2444         # sockets are safe: they will recreated be on the target side post-migrate
2445         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2446         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2447     }
2448
2449     die "VM uses local resources\n" if $loc_res && !$noerr;
2450
2451     return $loc_res;
2452 }
2453
2454 # check if used storages are available on all nodes (use by migrate)
2455 sub check_storage_availability {
2456     my ($storecfg, $conf, $node) = @_;
2457
2458     foreach_drive($conf, sub {
2459         my ($ds, $drive) = @_;
2460
2461         my $volid = $drive->{file};
2462         return if !$volid;
2463
2464         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2465         return if !$sid;
2466
2467         # check if storage is available on both nodes
2468         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2469         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2470    });
2471 }
2472
2473 # list nodes where all VM images are available (used by has_feature API)
2474 sub shared_nodes {
2475     my ($conf, $storecfg) = @_;
2476
2477     my $nodelist = PVE::Cluster::get_nodelist();
2478     my $nodehash = { map { $_ => 1 } @$nodelist };
2479     my $nodename = PVE::INotify::nodename();
2480
2481     foreach_drive($conf, sub {
2482         my ($ds, $drive) = @_;
2483
2484         my $volid = $drive->{file};
2485         return if !$volid;
2486
2487         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2488         if ($storeid) {
2489             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2490             if ($scfg->{disable}) {
2491                 $nodehash = {};
2492             } elsif (my $avail = $scfg->{nodes}) {
2493                 foreach my $node (keys %$nodehash) {
2494                     delete $nodehash->{$node} if !$avail->{$node};
2495                 }
2496             } elsif (!$scfg->{shared}) {
2497                 foreach my $node (keys %$nodehash) {
2498                     delete $nodehash->{$node} if $node ne $nodename
2499                 }
2500             }
2501         }
2502     });
2503
2504     return $nodehash
2505 }
2506
2507 sub check_cmdline {
2508     my ($pidfile, $pid) = @_;
2509
2510     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2511     if (defined($fh)) {
2512         my $line = <$fh>;
2513         $fh->close;
2514         return undef if !$line;
2515         my @param = split(/\0/, $line);
2516
2517         my $cmd = $param[0];
2518         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2519
2520         for (my $i = 0; $i < scalar (@param); $i++) {
2521             my $p = $param[$i];
2522             next if !$p;
2523             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2524                 my $p = $param[$i+1];
2525                 return 1 if $p && ($p eq $pidfile);
2526                 return undef;
2527             }
2528         }
2529     }
2530     return undef;
2531 }
2532
2533 sub check_running {
2534     my ($vmid, $nocheck, $node) = @_;
2535
2536     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2537
2538     die "unable to find configuration file for VM $vmid - no such machine\n"
2539         if !$nocheck && ! -f $filename;
2540
2541     my $pidfile = pidfile_name($vmid);
2542
2543     if (my $fd = IO::File->new("<$pidfile")) {
2544         my $st = stat($fd);
2545         my $line = <$fd>;
2546         close($fd);
2547
2548         my $mtime = $st->mtime;
2549         if ($mtime > time()) {
2550             warn "file '$filename' modified in future\n";
2551         }
2552
2553         if ($line =~ m/^(\d+)$/) {
2554             my $pid = $1;
2555             if (check_cmdline($pidfile, $pid)) {
2556                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2557                     return $pid;
2558                 }
2559             }
2560         }
2561     }
2562
2563     return undef;
2564 }
2565
2566 sub vzlist {
2567
2568     my $vzlist = config_list();
2569
2570     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2571
2572     while (defined(my $de = $fd->read)) {
2573         next if $de !~ m/^(\d+)\.pid$/;
2574         my $vmid = $1;
2575         next if !defined($vzlist->{$vmid});
2576         if (my $pid = check_running($vmid)) {
2577             $vzlist->{$vmid}->{pid} = $pid;
2578         }
2579     }
2580
2581     return $vzlist;
2582 }
2583
2584 sub disksize {
2585     my ($storecfg, $conf) = @_;
2586
2587     my $bootdisk = $conf->{bootdisk};
2588     return undef if !$bootdisk;
2589     return undef if !is_valid_drivename($bootdisk);
2590
2591     return undef if !$conf->{$bootdisk};
2592
2593     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2594     return undef if !defined($drive);
2595
2596     return undef if drive_is_cdrom($drive);
2597
2598     my $volid = $drive->{file};
2599     return undef if !$volid;
2600
2601     return $drive->{size};
2602 }
2603
2604 my $last_proc_pid_stat;
2605
2606 # get VM status information
2607 # This must be fast and should not block ($full == false)
2608 # We only query KVM using QMP if $full == true (this can be slow)
2609 sub vmstatus {
2610     my ($opt_vmid, $full) = @_;
2611
2612     my $res = {};
2613
2614     my $storecfg = PVE::Storage::config();
2615
2616     my $list = vzlist();
2617     my $defaults = load_defaults();
2618
2619     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2620
2621     my $cpucount = $cpuinfo->{cpus} || 1;
2622
2623     foreach my $vmid (keys %$list) {
2624         next if $opt_vmid && ($vmid ne $opt_vmid);
2625
2626         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2627         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2628
2629         my $d = {};
2630         $d->{pid} = $list->{$vmid}->{pid};
2631
2632         # fixme: better status?
2633         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2634
2635         my $size = disksize($storecfg, $conf);
2636         if (defined($size)) {
2637             $d->{disk} = 0; # no info available
2638             $d->{maxdisk} = $size;
2639         } else {
2640             $d->{disk} = 0;
2641             $d->{maxdisk} = 0;
2642         }
2643
2644         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2645             * ($conf->{cores} || $defaults->{cores});
2646         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2647         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2648
2649         $d->{name} = $conf->{name} || "VM $vmid";
2650         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2651             : $defaults->{memory}*(1024*1024);
2652
2653         if ($conf->{balloon}) {
2654             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2655             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2656                 : $defaults->{shares};
2657         }
2658
2659         $d->{uptime} = 0;
2660         $d->{cpu} = 0;
2661         $d->{mem} = 0;
2662
2663         $d->{netout} = 0;
2664         $d->{netin} = 0;
2665
2666         $d->{diskread} = 0;
2667         $d->{diskwrite} = 0;
2668
2669         $d->{template} = PVE::QemuConfig->is_template($conf);
2670
2671         $res->{$vmid} = $d;
2672     }
2673
2674     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2675     foreach my $dev (keys %$netdev) {
2676         next if $dev !~ m/^tap([1-9]\d*)i/;
2677         my $vmid = $1;
2678         my $d = $res->{$vmid};
2679         next if !$d;
2680
2681         $d->{netout} += $netdev->{$dev}->{receive};
2682         $d->{netin} += $netdev->{$dev}->{transmit};
2683
2684         if ($full) {
2685             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2686             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2687         }
2688
2689     }
2690
2691     my $ctime = gettimeofday;
2692
2693     foreach my $vmid (keys %$list) {
2694
2695         my $d = $res->{$vmid};
2696         my $pid = $d->{pid};
2697         next if !$pid;
2698
2699         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2700         next if !$pstat; # not running
2701
2702         my $used = $pstat->{utime} + $pstat->{stime};
2703
2704         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2705
2706         if ($pstat->{vsize}) {
2707             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2708         }
2709
2710         my $old = $last_proc_pid_stat->{$pid};
2711         if (!$old) {
2712             $last_proc_pid_stat->{$pid} = {
2713                 time => $ctime,
2714                 used => $used,
2715                 cpu => 0,
2716             };
2717             next;
2718         }
2719
2720         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2721
2722         if ($dtime > 1000) {
2723             my $dutime = $used -  $old->{used};
2724
2725             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2726             $last_proc_pid_stat->{$pid} = {
2727                 time => $ctime,
2728                 used => $used,
2729                 cpu => $d->{cpu},
2730             };
2731         } else {
2732             $d->{cpu} = $old->{cpu};
2733         }
2734     }
2735
2736     return $res if !$full;
2737
2738     my $qmpclient = PVE::QMPClient->new();
2739
2740     my $ballooncb = sub {
2741         my ($vmid, $resp) = @_;
2742
2743         my $info = $resp->{'return'};
2744         return if !$info->{max_mem};
2745
2746         my $d = $res->{$vmid};
2747
2748         # use memory assigned to VM
2749         $d->{maxmem} = $info->{max_mem};
2750         $d->{balloon} = $info->{actual};
2751
2752         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2753             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2754             $d->{freemem} = $info->{free_mem};
2755         }
2756
2757         $d->{ballooninfo} = $info;
2758     };
2759
2760     my $blockstatscb = sub {
2761         my ($vmid, $resp) = @_;
2762         my $data = $resp->{'return'} || [];
2763         my $totalrdbytes = 0;
2764         my $totalwrbytes = 0;
2765
2766         for my $blockstat (@$data) {
2767             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2768             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2769
2770             $blockstat->{device} =~ s/drive-//;
2771             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2772         }
2773         $res->{$vmid}->{diskread} = $totalrdbytes;
2774         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2775     };
2776
2777     my $statuscb = sub {
2778         my ($vmid, $resp) = @_;
2779
2780         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2781         # this fails if ballon driver is not loaded, so this must be
2782         # the last commnand (following command are aborted if this fails).
2783         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2784
2785         my $status = 'unknown';
2786         if (!defined($status = $resp->{'return'}->{status})) {
2787             warn "unable to get VM status\n";
2788             return;
2789         }
2790
2791         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2792     };
2793
2794     foreach my $vmid (keys %$list) {
2795         next if $opt_vmid && ($vmid ne $opt_vmid);
2796         next if !$res->{$vmid}->{pid}; # not running
2797         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2798     }
2799
2800     $qmpclient->queue_execute(undef, 2);
2801
2802     foreach my $vmid (keys %$list) {
2803         next if $opt_vmid && ($vmid ne $opt_vmid);
2804         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2805     }
2806
2807     return $res;
2808 }
2809
2810 sub foreach_drive {
2811     my ($conf, $func, @param) = @_;
2812
2813     foreach my $ds (valid_drive_names()) {
2814         next if !defined($conf->{$ds});
2815
2816         my $drive = parse_drive($ds, $conf->{$ds});
2817         next if !$drive;
2818
2819         &$func($ds, $drive, @param);
2820     }
2821 }
2822
2823 sub foreach_volid {
2824     my ($conf, $func, @param) = @_;
2825
2826     my $volhash = {};
2827
2828     my $test_volid = sub {
2829         my ($volid, $is_cdrom, $replicate, $snapname) = @_;
2830
2831         return if !$volid;
2832
2833         $volhash->{$volid}->{cdrom} //= 1;
2834         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2835
2836         $volhash->{$volid}->{replicate} //= 0;
2837         $volhash->{$volid}->{replicate} = 1 if $replicate;
2838
2839         $volhash->{$volid}->{referenced_in_config} //= 0;
2840         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
2841
2842         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
2843             if defined($snapname);
2844     };
2845
2846     foreach_drive($conf, sub {
2847         my ($ds, $drive) = @_;
2848         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, undef);
2849     });
2850
2851     foreach my $snapname (keys %{$conf->{snapshots}}) {
2852         my $snap = $conf->{snapshots}->{$snapname};
2853         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
2854         foreach_drive($snap, sub {
2855             my ($ds, $drive) = @_;
2856             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $snapname);
2857         });
2858     }
2859
2860     foreach my $volid (keys %$volhash) {
2861         &$func($volid, $volhash->{$volid}, @param);
2862     }
2863 }
2864
2865 sub vga_conf_has_spice {
2866     my ($vga) = @_;
2867
2868     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2869
2870     return $1 || 1;
2871 }
2872
2873 sub config_to_command {
2874     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2875
2876     my $cmd = [];
2877     my $globalFlags = [];
2878     my $machineFlags = [];
2879     my $rtcFlags = [];
2880     my $cpuFlags = [];
2881     my $devices = [];
2882     my $pciaddr = '';
2883     my $bridges = {};
2884     my $kvmver = kvm_user_version();
2885     my $vernum = 0; # unknown
2886     my $ostype = $conf->{ostype};
2887     my $winversion = windows_version($ostype);
2888     my $kvm = $conf->{kvm} // 1;
2889
2890     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
2891
2892     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2893         $vernum = $1*1000000+$2*1000;
2894     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2895         $vernum = $1*1000000+$2*1000+$3;
2896     }
2897
2898     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2899
2900     my $have_ovz = -f '/proc/vz/vestat';
2901
2902     my $q35 = machine_type_is_q35($conf);
2903     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2904     my $machine_type = $forcemachine || $conf->{machine};
2905     my $use_old_bios_files = undef;
2906     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2907
2908     my $cpuunits = defined($conf->{cpuunits}) ?
2909             $conf->{cpuunits} : $defaults->{cpuunits};
2910
2911     push @$cmd, '/usr/bin/kvm';
2912
2913     push @$cmd, '-id', $vmid;
2914
2915     my $use_virtio = 0;
2916
2917     my $qmpsocket = qmp_socket($vmid);
2918     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2919     push @$cmd, '-mon', "chardev=qmp,mode=control";
2920
2921
2922     push @$cmd, '-pidfile' , pidfile_name($vmid);
2923
2924     push @$cmd, '-daemonize';
2925
2926     if ($conf->{smbios1}) {
2927         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2928     }
2929
2930     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2931         die "uefi base image not found\n" if ! -f $OVMF_CODE;
2932
2933         my $path;
2934         my $format;
2935         if (my $efidisk = $conf->{efidisk0}) {
2936             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
2937             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2938             $format = $d->{format};
2939             if ($storeid) {
2940                 $path = PVE::Storage::path($storecfg, $d->{file});
2941                 if (!defined($format)) {
2942                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2943                     $format = qemu_img_format($scfg, $volname);
2944                 }
2945             } else {
2946                 $path = $d->{file};
2947                 die "efidisk format must be specified\n"
2948                     if !defined($format);
2949             }
2950         } else {
2951             warn "no efidisk configured! Using temporary efivars disk.\n";
2952             $path = "/tmp/$vmid-ovmf.fd";
2953             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
2954             $format = 'raw';
2955         }
2956
2957         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
2958         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
2959     }
2960
2961
2962     # add usb controllers
2963     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2964     push @$devices, @usbcontrollers if @usbcontrollers;
2965     my $vga = $conf->{vga};
2966
2967     my $qxlnum = vga_conf_has_spice($vga);
2968     $vga = 'qxl' if $qxlnum;
2969
2970     if (!$vga) {
2971         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
2972             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
2973         } else {
2974             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
2975         }
2976     }
2977
2978     # enable absolute mouse coordinates (needed by vnc)
2979     my $tablet;
2980     if (defined($conf->{tablet})) {
2981         $tablet = $conf->{tablet};
2982     } else {
2983         $tablet = $defaults->{tablet};
2984         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2985         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2986     }
2987
2988     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2989
2990     my $kvm_off = 0;
2991     my $gpu_passthrough;
2992
2993     # host pci devices
2994     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2995         my $d = parse_hostpci($conf->{"hostpci$i"});
2996         next if !$d;
2997
2998         my $pcie = $d->{pcie};
2999         if($pcie){
3000             die "q35 machine model is not enabled" if !$q35;
3001             $pciaddr = print_pcie_addr("hostpci$i");
3002         }else{
3003             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3004         }
3005
3006         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3007         my $romfile = $d->{romfile};
3008
3009         my $xvga = '';
3010         if ($d->{'x-vga'}) {
3011             $xvga = ',x-vga=on';
3012             $kvm_off = 1;
3013             $vga = 'none';
3014             $gpu_passthrough = 1;
3015
3016             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3017                 $xvga = "";
3018             }
3019         }
3020         my $pcidevices = $d->{pciid};
3021         my $multifunction = 1 if @$pcidevices > 1;
3022
3023         my $j=0;
3024         foreach my $pcidevice (@$pcidevices) {
3025
3026             my $id = "hostpci$i";
3027             $id .= ".$j" if $multifunction;
3028             my $addr = $pciaddr;
3029             $addr .= ".$j" if $multifunction;
3030             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3031
3032             if($j == 0){
3033                 $devicestr .= "$rombar$xvga";
3034                 $devicestr .= ",multifunction=on" if $multifunction;
3035                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3036             }
3037
3038             push @$devices, '-device', $devicestr;
3039             $j++;
3040         }
3041     }
3042
3043     # usb devices
3044     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3045     push @$devices, @usbdevices if @usbdevices;
3046     # serial devices
3047     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3048         if (my $path = $conf->{"serial$i"}) {
3049             if ($path eq 'socket') {
3050                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3051                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3052                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3053             } else {
3054                 die "no such serial device\n" if ! -c $path;
3055                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3056                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3057             }
3058         }
3059     }
3060
3061     # parallel devices
3062     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3063         if (my $path = $conf->{"parallel$i"}) {
3064             die "no such parallel device\n" if ! -c $path;
3065             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3066             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3067             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3068         }
3069     }
3070
3071     my $vmname = $conf->{name} || "vm$vmid";
3072
3073     push @$cmd, '-name', $vmname;
3074
3075     my $sockets = 1;
3076     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3077     $sockets = $conf->{sockets} if  $conf->{sockets};
3078
3079     my $cores = $conf->{cores} || 1;
3080
3081     my $maxcpus = $sockets * $cores;
3082
3083     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3084
3085     my $allowed_vcpus = $cpuinfo->{cpus};
3086
3087     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3088         if ($allowed_vcpus < $maxcpus);
3089
3090     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3091
3092         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3093         for (my $i = 2; $i <= $vcpus; $i++)  {
3094             my $cpustr = print_cpu_device($conf,$i);
3095             push @$cmd, '-device', $cpustr;
3096         }
3097
3098     } else {
3099
3100         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3101     }
3102     push @$cmd, '-nodefaults';
3103
3104     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3105
3106     my $bootindex_hash = {};
3107     my $i = 1;
3108     foreach my $o (split(//, $bootorder)) {
3109         $bootindex_hash->{$o} = $i*100;
3110         $i++;
3111     }
3112
3113     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3114
3115     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3116
3117     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3118
3119     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3120
3121     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3122         my $socket = vnc_socket($vmid);
3123         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3124     } else {
3125         push @$cmd, '-nographic';
3126     }
3127
3128     # time drift fix
3129     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3130
3131     my $useLocaltime = $conf->{localtime};
3132
3133     if ($winversion >= 5) { # windows
3134         $useLocaltime = 1 if !defined($conf->{localtime});
3135
3136         # use time drift fix when acpi is enabled
3137         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3138             $tdf = 1 if !defined($conf->{tdf});
3139         }
3140     }
3141
3142     if ($winversion >= 6) {
3143         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3144         push @$cmd, '-no-hpet';
3145     }
3146
3147     push @$rtcFlags, 'driftfix=slew' if $tdf;
3148
3149     if (!$kvm) {
3150         push @$machineFlags, 'accel=tcg';
3151     }
3152
3153     if ($machine_type) {
3154         push @$machineFlags, "type=${machine_type}";
3155     }
3156
3157     if ($conf->{startdate}) {
3158         push @$rtcFlags, "base=$conf->{startdate}";
3159     } elsif ($useLocaltime) {
3160         push @$rtcFlags, 'base=localtime';
3161     }
3162
3163     my $cpu = $kvm ? "kvm64" : "qemu64";
3164     if (my $cputype = $conf->{cpu}) {
3165         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3166             or die "Cannot parse cpu description: $cputype\n";
3167         $cpu = $cpuconf->{cputype};
3168         $kvm_off = 1 if $cpuconf->{hidden};
3169
3170         if (defined(my $flags = $cpuconf->{flags})) {
3171             push @$cpuFlags, split(";", $flags);
3172         }
3173     }
3174
3175     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3176
3177     push @$cpuFlags , '-x2apic'
3178         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3179
3180     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3181
3182     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3183
3184     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3185
3186         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3187         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3188     }
3189
3190     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3191
3192     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3193
3194     push @$cpuFlags, 'kvm=off' if $kvm_off;
3195
3196     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3197         die "internal error"; # should not happen
3198
3199     push @$cpuFlags, "vendor=${cpu_vendor}"
3200         if $cpu_vendor ne 'default';
3201
3202     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3203
3204     push @$cmd, '-cpu', $cpu;
3205
3206     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3207
3208     push @$cmd, '-S' if $conf->{freeze};
3209
3210     # set keyboard layout
3211     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3212     push @$cmd, '-k', $kb if $kb;
3213
3214     # enable sound
3215     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3216     #push @$cmd, '-soundhw', 'es1370';
3217     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3218
3219     if($conf->{agent}) {
3220         my $qgasocket = qmp_socket($vmid, 1);
3221         my $pciaddr = print_pci_addr("qga0", $bridges);
3222         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3223         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3224         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3225     }
3226
3227     my $spice_port;
3228
3229     if ($qxlnum) {
3230         if ($qxlnum > 1) {
3231             if ($winversion){
3232                 for(my $i = 1; $i < $qxlnum; $i++){
3233                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3234                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3235                 }
3236             } else {
3237                 # assume other OS works like Linux
3238                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3239                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3240             }
3241         }
3242
3243         my $pciaddr = print_pci_addr("spice", $bridges);
3244
3245         my $nodename = PVE::INotify::nodename();
3246         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3247         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3248         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3249         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3250         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3251
3252         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3253
3254         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3255         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3256         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3257     }
3258
3259     # enable balloon by default, unless explicitly disabled
3260     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3261         $pciaddr = print_pci_addr("balloon0", $bridges);
3262         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3263     }
3264
3265     if ($conf->{watchdog}) {
3266         my $wdopts = parse_watchdog($conf->{watchdog});
3267         $pciaddr = print_pci_addr("watchdog", $bridges);
3268         my $watchdog = $wdopts->{model} || 'i6300esb';
3269         push @$devices, '-device', "$watchdog$pciaddr";
3270         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3271     }
3272
3273     my $vollist = [];
3274     my $scsicontroller = {};
3275     my $ahcicontroller = {};
3276     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3277
3278     # Add iscsi initiator name if available
3279     if (my $initiator = get_initiator_name()) {
3280         push @$devices, '-iscsi', "initiator-name=$initiator";
3281     }
3282
3283     foreach_drive($conf, sub {
3284         my ($ds, $drive) = @_;
3285
3286         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3287             push @$vollist, $drive->{file};
3288         }
3289
3290         # ignore efidisk here, already added in bios/fw handling code above
3291         return if $drive->{interface} eq 'efidisk';
3292
3293         $use_virtio = 1 if $ds =~ m/^virtio/;
3294
3295         if (drive_is_cdrom ($drive)) {
3296             if ($bootindex_hash->{d}) {
3297                 $drive->{bootindex} = $bootindex_hash->{d};
3298                 $bootindex_hash->{d} += 1;
3299             }
3300         } else {
3301             if ($bootindex_hash->{c}) {
3302                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3303                 $bootindex_hash->{c} += 1;
3304             }
3305         }
3306
3307         if($drive->{interface} eq 'virtio'){
3308            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3309         }
3310
3311         if ($drive->{interface} eq 'scsi') {
3312
3313             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3314
3315             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3316             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3317
3318             my $iothread = '';
3319             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3320                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3321                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3322             } elsif ($drive->{iothread}) {
3323                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3324             }
3325
3326             my $queues = '';
3327             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3328                 $queues = ",num_queues=$drive->{queues}";
3329             }
3330
3331             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3332             $scsicontroller->{$controller}=1;
3333         }
3334
3335         if ($drive->{interface} eq 'sata') {
3336            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3337            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3338            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3339            $ahcicontroller->{$controller}=1;
3340         }
3341
3342         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3343         push @$devices, '-drive',$drive_cmd;
3344         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3345     });
3346
3347     for (my $i = 0; $i < $MAX_NETS; $i++) {
3348          next if !$conf->{"net$i"};
3349          my $d = parse_net($conf->{"net$i"});
3350          next if !$d;
3351
3352          $use_virtio = 1 if $d->{model} eq 'virtio';
3353
3354          if ($bootindex_hash->{n}) {
3355             $d->{bootindex} = $bootindex_hash->{n};
3356             $bootindex_hash->{n} += 1;
3357          }
3358
3359          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3360          push @$devices, '-netdev', $netdevfull;
3361
3362          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3363          push @$devices, '-device', $netdevicefull;
3364     }
3365
3366     if (!$q35) {
3367         # add pci bridges
3368         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3369            $bridges->{1} = 1;
3370            $bridges->{2} = 1;
3371         }
3372
3373         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3374
3375         while (my ($k, $v) = each %$bridges) {
3376             $pciaddr = print_pci_addr("pci.$k");
3377             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3378         }
3379     }
3380
3381     # add custom args
3382     if ($conf->{args}) {
3383         my $aa = PVE::Tools::split_args($conf->{args});
3384         push @$cmd, @$aa;
3385     }
3386
3387     push @$cmd, @$devices;
3388     push @$cmd, '-rtc', join(',', @$rtcFlags)
3389         if scalar(@$rtcFlags);
3390     push @$cmd, '-machine', join(',', @$machineFlags)
3391         if scalar(@$machineFlags);
3392     push @$cmd, '-global', join(',', @$globalFlags)
3393         if scalar(@$globalFlags);
3394
3395     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3396 }
3397
3398 sub vnc_socket {
3399     my ($vmid) = @_;
3400     return "${var_run_tmpdir}/$vmid.vnc";
3401 }
3402
3403 sub spice_port {
3404     my ($vmid) = @_;
3405
3406     my $res = vm_mon_cmd($vmid, 'query-spice');
3407
3408     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3409 }
3410
3411 sub qmp_socket {
3412     my ($vmid, $qga) = @_;
3413     my $sockettype = $qga ? 'qga' : 'qmp';
3414     return "${var_run_tmpdir}/$vmid.$sockettype";
3415 }
3416
3417 sub pidfile_name {
3418     my ($vmid) = @_;
3419     return "${var_run_tmpdir}/$vmid.pid";
3420 }
3421
3422 sub vm_devices_list {
3423     my ($vmid) = @_;
3424
3425     my $res = vm_mon_cmd($vmid, 'query-pci');
3426     my $devices = {};
3427     foreach my $pcibus (@$res) {
3428         foreach my $device (@{$pcibus->{devices}}) {
3429             next if !$device->{'qdev_id'};
3430             if ($device->{'pci_bridge'}) {
3431                 $devices->{$device->{'qdev_id'}} = 1;
3432                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3433                     next if !$bridge_device->{'qdev_id'};
3434                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3435                     $devices->{$device->{'qdev_id'}}++;
3436                 }
3437             } else {
3438                 $devices->{$device->{'qdev_id'}} = 1;
3439             }
3440         }
3441     }
3442
3443     my $resblock = vm_mon_cmd($vmid, 'query-block');
3444     foreach my $block (@$resblock) {
3445         if($block->{device} =~ m/^drive-(\S+)/){
3446                 $devices->{$1} = 1;
3447         }
3448     }
3449
3450     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3451     foreach my $mice (@$resmice) {
3452         if ($mice->{name} eq 'QEMU HID Tablet') {
3453             $devices->{tablet} = 1;
3454             last;
3455         }
3456     }
3457
3458     # for usb devices there is no query-usb
3459     # but we can iterate over the entries in
3460     # qom-list path=/machine/peripheral
3461     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3462     foreach my $per (@$resperipheral) {
3463         if ($per->{name} =~ m/^usb\d+$/) {
3464             $devices->{$per->{name}} = 1;
3465         }
3466     }
3467
3468     return $devices;
3469 }
3470
3471 sub vm_deviceplug {
3472     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3473
3474     my $q35 = machine_type_is_q35($conf);
3475
3476     my $devices_list = vm_devices_list($vmid);
3477     return 1 if defined($devices_list->{$deviceid});
3478
3479     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3480
3481     if ($deviceid eq 'tablet') {
3482
3483         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3484
3485     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3486
3487         die "usb hotplug currently not reliable\n";
3488         # since we can't reliably hot unplug all added usb devices
3489         # and usb passthrough disables live migration
3490         # we disable usb hotplugging for now
3491         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3492
3493     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3494
3495         qemu_iothread_add($vmid, $deviceid, $device);
3496
3497         qemu_driveadd($storecfg, $vmid, $device);
3498         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3499
3500         qemu_deviceadd($vmid, $devicefull);
3501         eval { qemu_deviceaddverify($vmid, $deviceid); };
3502         if (my $err = $@) {
3503             eval { qemu_drivedel($vmid, $deviceid); };
3504             warn $@ if $@;
3505             die $err;
3506         }
3507
3508     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3509
3510
3511         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3512         my $pciaddr = print_pci_addr($deviceid);
3513         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3514
3515         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3516
3517         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3518             qemu_iothread_add($vmid, $deviceid, $device);
3519             $devicefull .= ",iothread=iothread-$deviceid";
3520         }
3521
3522         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3523             $devicefull .= ",num_queues=$device->{queues}";
3524         }
3525
3526         qemu_deviceadd($vmid, $devicefull);
3527         qemu_deviceaddverify($vmid, $deviceid);
3528
3529     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3530
3531         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3532         qemu_driveadd($storecfg, $vmid, $device);
3533
3534         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3535         eval { qemu_deviceadd($vmid, $devicefull); };
3536         if (my $err = $@) {
3537             eval { qemu_drivedel($vmid, $deviceid); };
3538             warn $@ if $@;
3539             die $err;
3540         }
3541
3542     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3543
3544         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3545
3546         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3547         my $use_old_bios_files = undef;
3548         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3549
3550         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3551         qemu_deviceadd($vmid, $netdevicefull);
3552         eval { qemu_deviceaddverify($vmid, $deviceid); };
3553         if (my $err = $@) {
3554             eval { qemu_netdevdel($vmid, $deviceid); };
3555             warn $@ if $@;
3556             die $err;
3557         }
3558
3559     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3560
3561         my $bridgeid = $2;
3562         my $pciaddr = print_pci_addr($deviceid);
3563         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3564
3565         qemu_deviceadd($vmid, $devicefull);
3566         qemu_deviceaddverify($vmid, $deviceid);
3567
3568     } else {
3569         die "can't hotplug device '$deviceid'\n";
3570     }
3571
3572     return 1;
3573 }
3574
3575 # fixme: this should raise exceptions on error!
3576 sub vm_deviceunplug {
3577     my ($vmid, $conf, $deviceid) = @_;
3578
3579     my $devices_list = vm_devices_list($vmid);
3580     return 1 if !defined($devices_list->{$deviceid});
3581
3582     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3583
3584     if ($deviceid eq 'tablet') {
3585
3586         qemu_devicedel($vmid, $deviceid);
3587
3588     } elsif ($deviceid =~ m/^usb\d+$/) {
3589
3590         die "usb hotplug currently not reliable\n";
3591         # when unplugging usb devices this way,
3592         # there may be remaining usb controllers/hubs
3593         # so we disable it for now
3594         qemu_devicedel($vmid, $deviceid);
3595         qemu_devicedelverify($vmid, $deviceid);
3596
3597     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3598
3599         qemu_devicedel($vmid, $deviceid);
3600         qemu_devicedelverify($vmid, $deviceid);
3601         qemu_drivedel($vmid, $deviceid);
3602         qemu_iothread_del($conf, $vmid, $deviceid);
3603
3604     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3605
3606         qemu_devicedel($vmid, $deviceid);
3607         qemu_devicedelverify($vmid, $deviceid);
3608         qemu_iothread_del($conf, $vmid, $deviceid);
3609
3610     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3611
3612         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3613         my $device = parse_drive($deviceid, $conf->{$deviceid});
3614         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3615
3616         qemu_devicedel($vmid, $deviceid);
3617         qemu_drivedel($vmid, $deviceid);
3618         qemu_deletescsihw($conf, $vmid, $deviceid);
3619
3620     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3621
3622         qemu_devicedel($vmid, $deviceid);
3623         qemu_devicedelverify($vmid, $deviceid);
3624         qemu_netdevdel($vmid, $deviceid);
3625
3626     } else {
3627         die "can't unplug device '$deviceid'\n";
3628     }
3629
3630     return 1;
3631 }
3632
3633 sub qemu_deviceadd {
3634     my ($vmid, $devicefull) = @_;
3635
3636     $devicefull = "driver=".$devicefull;
3637     my %options =  split(/[=,]/, $devicefull);
3638
3639     vm_mon_cmd($vmid, "device_add" , %options);
3640 }
3641
3642 sub qemu_devicedel {
3643     my ($vmid, $deviceid) = @_;
3644
3645     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3646 }
3647
3648 sub qemu_iothread_add {
3649     my($vmid, $deviceid, $device) = @_;
3650
3651     if ($device->{iothread}) {
3652         my $iothreads = vm_iothreads_list($vmid);
3653         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3654     }
3655 }
3656
3657 sub qemu_iothread_del {
3658     my($conf, $vmid, $deviceid) = @_;
3659
3660     my $device = parse_drive($deviceid, $conf->{$deviceid});
3661     if ($device->{iothread}) {
3662         my $iothreads = vm_iothreads_list($vmid);
3663         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3664     }
3665 }
3666
3667 sub qemu_objectadd {
3668     my($vmid, $objectid, $qomtype) = @_;
3669
3670     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3671
3672     return 1;
3673 }
3674
3675 sub qemu_objectdel {
3676     my($vmid, $objectid) = @_;
3677
3678     vm_mon_cmd($vmid, "object-del", id => $objectid);
3679
3680     return 1;
3681 }
3682
3683 sub qemu_driveadd {
3684     my ($storecfg, $vmid, $device) = @_;
3685
3686     my $drive = print_drive_full($storecfg, $vmid, $device);
3687     $drive =~ s/\\/\\\\/g;
3688     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3689
3690     # If the command succeeds qemu prints: "OK"
3691     return 1 if $ret =~ m/OK/s;
3692
3693     die "adding drive failed: $ret\n";
3694 }
3695
3696 sub qemu_drivedel {
3697     my($vmid, $deviceid) = @_;
3698
3699     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3700     $ret =~ s/^\s+//;
3701
3702     return 1 if $ret eq "";
3703
3704     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3705     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3706
3707     die "deleting drive $deviceid failed : $ret\n";
3708 }
3709
3710 sub qemu_deviceaddverify {
3711     my ($vmid, $deviceid) = @_;
3712
3713     for (my $i = 0; $i <= 5; $i++) {
3714          my $devices_list = vm_devices_list($vmid);
3715          return 1 if defined($devices_list->{$deviceid});
3716          sleep 1;
3717     }
3718
3719     die "error on hotplug device '$deviceid'\n";
3720 }
3721
3722
3723 sub qemu_devicedelverify {
3724     my ($vmid, $deviceid) = @_;
3725
3726     # need to verify that the device is correctly removed as device_del
3727     # is async and empty return is not reliable
3728
3729     for (my $i = 0; $i <= 5; $i++) {
3730          my $devices_list = vm_devices_list($vmid);
3731          return 1 if !defined($devices_list->{$deviceid});
3732          sleep 1;
3733     }
3734
3735     die "error on hot-unplugging device '$deviceid'\n";
3736 }
3737
3738 sub qemu_findorcreatescsihw {
3739     my ($storecfg, $conf, $vmid, $device) = @_;
3740
3741     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3742
3743     my $scsihwid="$controller_prefix$controller";
3744     my $devices_list = vm_devices_list($vmid);
3745
3746     if(!defined($devices_list->{$scsihwid})) {
3747         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3748     }
3749
3750     return 1;
3751 }
3752
3753 sub qemu_deletescsihw {
3754     my ($conf, $vmid, $opt) = @_;
3755
3756     my $device = parse_drive($opt, $conf->{$opt});
3757
3758     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3759         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3760         return 1;
3761     }
3762
3763     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3764
3765     my $devices_list = vm_devices_list($vmid);
3766     foreach my $opt (keys %{$devices_list}) {
3767         if (PVE::QemuServer::is_valid_drivename($opt)) {
3768             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3769             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3770                 return 1;
3771             }
3772         }
3773     }
3774
3775     my $scsihwid="scsihw$controller";
3776
3777     vm_deviceunplug($vmid, $conf, $scsihwid);
3778
3779     return 1;
3780 }
3781
3782 sub qemu_add_pci_bridge {
3783     my ($storecfg, $conf, $vmid, $device) = @_;
3784
3785     my $bridges = {};
3786
3787     my $bridgeid;
3788
3789     print_pci_addr($device, $bridges);
3790
3791     while (my ($k, $v) = each %$bridges) {
3792         $bridgeid = $k;
3793     }
3794     return 1 if !defined($bridgeid) || $bridgeid < 1;
3795
3796     my $bridge = "pci.$bridgeid";
3797     my $devices_list = vm_devices_list($vmid);
3798
3799     if (!defined($devices_list->{$bridge})) {
3800         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3801     }
3802
3803     return 1;
3804 }
3805
3806 sub qemu_set_link_status {
3807     my ($vmid, $device, $up) = @_;
3808
3809     vm_mon_cmd($vmid, "set_link", name => $device,
3810                up => $up ? JSON::true : JSON::false);
3811 }
3812
3813 sub qemu_netdevadd {
3814     my ($vmid, $conf, $device, $deviceid) = @_;
3815
3816     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3817     my %options =  split(/[=,]/, $netdev);
3818
3819     vm_mon_cmd($vmid, "netdev_add",  %options);
3820     return 1;
3821 }
3822
3823 sub qemu_netdevdel {
3824     my ($vmid, $deviceid) = @_;
3825
3826     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3827 }
3828
3829 sub qemu_usb_hotplug {
3830     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3831
3832     return if !$device;
3833
3834     # remove the old one first
3835     vm_deviceunplug($vmid, $conf, $deviceid);
3836
3837     # check if xhci controller is necessary and available
3838     if ($device->{usb3}) {
3839
3840         my $devicelist = vm_devices_list($vmid);
3841
3842         if (!$devicelist->{xhci}) {
3843             my $pciaddr = print_pci_addr("xhci");
3844             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3845         }
3846     }
3847     my $d = parse_usb_device($device->{host});
3848     $d->{usb3} = $device->{usb3};
3849
3850     # add the new one
3851     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3852 }
3853
3854 sub qemu_cpu_hotplug {
3855     my ($vmid, $conf, $vcpus) = @_;
3856
3857     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3858
3859     my $sockets = 1;
3860     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3861     $sockets = $conf->{sockets} if  $conf->{sockets};
3862     my $cores = $conf->{cores} || 1;
3863     my $maxcpus = $sockets * $cores;
3864
3865     $vcpus = $maxcpus if !$vcpus;
3866
3867     die "you can't add more vcpus than maxcpus\n"
3868         if $vcpus > $maxcpus;
3869
3870     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3871
3872     if ($vcpus < $currentvcpus) {
3873
3874         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3875
3876             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3877                 qemu_devicedel($vmid, "cpu$i");
3878                 my $retry = 0;
3879                 my $currentrunningvcpus = undef;
3880                 while (1) {
3881                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3882                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3883                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3884                     $retry++;
3885                     sleep 1;
3886                 }
3887                 #update conf after each succesfull cpu unplug
3888                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3889                 PVE::QemuConfig->write_config($vmid, $conf);
3890             }
3891         } else {
3892             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3893         }
3894
3895         return;
3896     }
3897
3898     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3899     die "vcpus in running vm does not match its configuration\n"
3900         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3901
3902     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3903
3904         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3905             my $cpustr = print_cpu_device($conf, $i);
3906             qemu_deviceadd($vmid, $cpustr);
3907
3908             my $retry = 0;
3909             my $currentrunningvcpus = undef;
3910             while (1) {
3911                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3912                 last if scalar(@{$currentrunningvcpus}) == $i;
3913                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3914                 sleep 1;
3915                 $retry++;
3916             }
3917             #update conf after each succesfull cpu hotplug
3918             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3919             PVE::QemuConfig->write_config($vmid, $conf);
3920         }
3921     } else {
3922
3923         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3924             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3925         }
3926     }
3927 }
3928
3929 sub qemu_block_set_io_throttle {
3930     my ($vmid, $deviceid,
3931         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3932         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3933         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3934         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3935
3936     return if !check_running($vmid) ;
3937
3938     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3939         bps => int($bps),
3940         bps_rd => int($bps_rd),
3941         bps_wr => int($bps_wr),
3942         iops => int($iops),
3943         iops_rd => int($iops_rd),
3944         iops_wr => int($iops_wr),
3945         bps_max => int($bps_max),
3946         bps_rd_max => int($bps_rd_max),
3947         bps_wr_max => int($bps_wr_max),
3948         iops_max => int($iops_max),
3949         iops_rd_max => int($iops_rd_max),
3950         iops_wr_max => int($iops_wr_max),
3951         bps_max_length => int($bps_max_length),
3952         bps_rd_max_length => int($bps_rd_max_length),
3953         bps_wr_max_length => int($bps_wr_max_length),
3954         iops_max_length => int($iops_max_length),
3955         iops_rd_max_length => int($iops_rd_max_length),
3956         iops_wr_max_length => int($iops_wr_max_length),
3957     );
3958
3959 }
3960
3961 # old code, only used to shutdown old VM after update
3962 sub __read_avail {
3963     my ($fh, $timeout) = @_;
3964
3965     my $sel = new IO::Select;
3966     $sel->add($fh);
3967
3968     my $res = '';
3969     my $buf;
3970
3971     my @ready;
3972     while (scalar (@ready = $sel->can_read($timeout))) {
3973         my $count;
3974         if ($count = $fh->sysread($buf, 8192)) {
3975             if ($buf =~ /^(.*)\(qemu\) $/s) {
3976                 $res .= $1;
3977                 last;
3978             } else {
3979                 $res .= $buf;
3980             }
3981         } else {
3982             if (!defined($count)) {
3983                 die "$!\n";
3984             }
3985             last;
3986         }
3987     }
3988
3989     die "monitor read timeout\n" if !scalar(@ready);
3990
3991     return $res;
3992 }
3993
3994 # old code, only used to shutdown old VM after update
3995 sub vm_monitor_command {
3996     my ($vmid, $cmdstr, $nocheck) = @_;
3997
3998     my $res;
3999
4000     eval {
4001         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4002
4003         my $sname = "${var_run_tmpdir}/$vmid.mon";
4004
4005         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
4006             die "unable to connect to VM $vmid socket - $!\n";
4007
4008         my $timeout = 3;
4009
4010         # hack: migrate sometime blocks the monitor (when migrate_downtime
4011         # is set)
4012         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4013             $timeout = 60*60; # 1 hour
4014         }
4015
4016         # read banner;
4017         my $data = __read_avail($sock, $timeout);
4018
4019         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
4020             die "got unexpected qemu monitor banner\n";
4021         }
4022
4023         my $sel = new IO::Select;
4024         $sel->add($sock);
4025
4026         if (!scalar(my @ready = $sel->can_write($timeout))) {
4027             die "monitor write error - timeout";
4028         }
4029
4030         my $fullcmd = "$cmdstr\r";
4031
4032         # syslog('info', "VM $vmid monitor command: $cmdstr");
4033
4034         my $b;
4035         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
4036             die "monitor write error - $!";
4037         }
4038
4039         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4040
4041         $timeout = 20;
4042
4043         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4044             $timeout = 60*60; # 1 hour
4045         } elsif ($cmdstr =~ m/^(eject|change)/) {
4046             $timeout = 60; # note: cdrom mount command is slow
4047         }
4048         if ($res = __read_avail($sock, $timeout)) {
4049
4050             my @lines = split("\r?\n", $res);
4051
4052             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4053
4054             $res = join("\n", @lines);
4055             $res .= "\n";
4056         }
4057     };
4058
4059     my $err = $@;
4060
4061     if ($err) {
4062         syslog("err", "VM $vmid monitor command failed - $err");
4063         die $err;
4064     }
4065
4066     return $res;
4067 }
4068
4069 sub qemu_block_resize {
4070     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4071
4072     my $running = check_running($vmid);
4073
4074     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4075
4076     return if !$running;
4077
4078     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4079
4080 }
4081
4082 sub qemu_volume_snapshot {
4083     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4084
4085     my $running = check_running($vmid);
4086
4087     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4088         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4089     } else {
4090         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4091     }
4092 }
4093
4094 sub qemu_volume_snapshot_delete {
4095     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4096
4097     my $running = check_running($vmid);
4098
4099     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4100         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4101     } else {
4102         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4103     }
4104 }
4105
4106 sub set_migration_caps {
4107     my ($vmid) = @_;
4108
4109     my $cap_ref = [];
4110
4111     my $enabled_cap = {
4112         "auto-converge" => 1,
4113         "xbzrle" => 1,
4114         "x-rdma-pin-all" => 0,
4115         "zero-blocks" => 0,
4116         "compress" => 0
4117     };
4118
4119     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4120
4121     for my $supported_capability (@$supported_capabilities) {
4122         push @$cap_ref, {
4123             capability => $supported_capability->{capability},
4124             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4125         };
4126     }
4127
4128     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4129 }
4130
4131 my $fast_plug_option = {
4132     'lock' => 1,
4133     'name' => 1,
4134     'onboot' => 1,
4135     'shares' => 1,
4136     'startup' => 1,
4137     'description' => 1,
4138     'protection' => 1,
4139     'vmstatestorage' => 1,
4140 };
4141
4142 # hotplug changes in [PENDING]
4143 # $selection hash can be used to only apply specified options, for
4144 # example: { cores => 1 } (only apply changed 'cores')
4145 # $errors ref is used to return error messages
4146 sub vmconfig_hotplug_pending {
4147     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4148
4149     my $defaults = load_defaults();
4150
4151     # commit values which do not have any impact on running VM first
4152     # Note: those option cannot raise errors, we we do not care about
4153     # $selection and always apply them.
4154
4155     my $add_error = sub {
4156         my ($opt, $msg) = @_;
4157         $errors->{$opt} = "hotplug problem - $msg";
4158     };
4159
4160     my $changes = 0;
4161     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4162         if ($fast_plug_option->{$opt}) {
4163             $conf->{$opt} = $conf->{pending}->{$opt};
4164             delete $conf->{pending}->{$opt};
4165             $changes = 1;
4166         }
4167     }
4168
4169     if ($changes) {
4170         PVE::QemuConfig->write_config($vmid, $conf);
4171         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4172     }
4173
4174     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4175
4176     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4177     while (my ($opt, $force) = each %$pending_delete_hash) {
4178         next if $selection && !$selection->{$opt};
4179         eval {
4180             if ($opt eq 'hotplug') {
4181                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4182             } elsif ($opt eq 'tablet') {
4183                 die "skip\n" if !$hotplug_features->{usb};
4184                 if ($defaults->{tablet}) {
4185                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4186                 } else {
4187                     vm_deviceunplug($vmid, $conf, $opt);
4188                 }
4189             } elsif ($opt =~ m/^usb\d+/) {
4190                 die "skip\n";
4191                 # since we cannot reliably hot unplug usb devices
4192                 # we are disabling it
4193                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4194                 vm_deviceunplug($vmid, $conf, $opt);
4195             } elsif ($opt eq 'vcpus') {
4196                 die "skip\n" if !$hotplug_features->{cpu};
4197                 qemu_cpu_hotplug($vmid, $conf, undef);
4198             } elsif ($opt eq 'balloon') {
4199                 # enable balloon device is not hotpluggable
4200                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4201             } elsif ($fast_plug_option->{$opt}) {
4202                 # do nothing
4203             } elsif ($opt =~ m/^net(\d+)$/) {
4204                 die "skip\n" if !$hotplug_features->{network};
4205                 vm_deviceunplug($vmid, $conf, $opt);
4206             } elsif (is_valid_drivename($opt)) {
4207                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4208                 vm_deviceunplug($vmid, $conf, $opt);
4209                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4210             } elsif ($opt =~ m/^memory$/) {
4211                 die "skip\n" if !$hotplug_features->{memory};
4212                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4213             } elsif ($opt eq 'cpuunits') {
4214                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4215             } elsif ($opt eq 'cpulimit') {
4216                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4217             } else {
4218                 die "skip\n";
4219             }
4220         };
4221         if (my $err = $@) {
4222             &$add_error($opt, $err) if $err ne "skip\n";
4223         } else {
4224             # save new config if hotplug was successful
4225             delete $conf->{$opt};
4226             vmconfig_undelete_pending_option($conf, $opt);
4227             PVE::QemuConfig->write_config($vmid, $conf);
4228             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4229         }
4230     }
4231
4232     foreach my $opt (keys %{$conf->{pending}}) {
4233         next if $selection && !$selection->{$opt};
4234         my $value = $conf->{pending}->{$opt};
4235         eval {
4236             if ($opt eq 'hotplug') {
4237                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4238             } elsif ($opt eq 'tablet') {
4239                 die "skip\n" if !$hotplug_features->{usb};
4240                 if ($value == 1) {
4241                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4242                 } elsif ($value == 0) {
4243                     vm_deviceunplug($vmid, $conf, $opt);
4244                 }
4245             } elsif ($opt =~ m/^usb\d+$/) {
4246                 die "skip\n";
4247                 # since we cannot reliably hot unplug usb devices
4248                 # we are disabling it
4249                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4250                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4251                 die "skip\n" if !$d;
4252                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4253             } elsif ($opt eq 'vcpus') {
4254                 die "skip\n" if !$hotplug_features->{cpu};
4255                 qemu_cpu_hotplug($vmid, $conf, $value);
4256             } elsif ($opt eq 'balloon') {
4257                 # enable/disable balloning device is not hotpluggable
4258                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4259                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4260                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4261
4262                 # allow manual ballooning if shares is set to zero
4263                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4264                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4265                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4266                 }
4267             } elsif ($opt =~ m/^net(\d+)$/) {
4268                 # some changes can be done without hotplug
4269                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4270                                     $vmid, $opt, $value);
4271             } elsif (is_valid_drivename($opt)) {
4272                 # some changes can be done without hotplug
4273                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4274                                      $vmid, $opt, $value, 1);
4275             } elsif ($opt =~ m/^memory$/) { #dimms
4276                 die "skip\n" if !$hotplug_features->{memory};
4277                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4278             } elsif ($opt eq 'cpuunits') {
4279                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4280             } elsif ($opt eq 'cpulimit') {
4281                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4282                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4283             } else {
4284                 die "skip\n";  # skip non-hot-pluggable options
4285             }
4286         };
4287         if (my $err = $@) {
4288             &$add_error($opt, $err) if $err ne "skip\n";
4289         } else {
4290             # save new config if hotplug was successful
4291             $conf->{$opt} = $value;
4292             delete $conf->{pending}->{$opt};
4293             PVE::QemuConfig->write_config($vmid, $conf);
4294             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4295         }
4296     }
4297 }
4298
4299 sub try_deallocate_drive {
4300     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4301
4302     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4303         my $volid = $drive->{file};
4304         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4305             my $sid = PVE::Storage::parse_volume_id($volid);
4306             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4307
4308             # check if the disk is really unused
4309             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4310                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4311             PVE::Storage::vdisk_free($storecfg, $volid);
4312             return 1;
4313         } else {
4314             # If vm is not owner of this disk remove from config
4315             return 1;
4316         }
4317     }
4318
4319     return undef;
4320 }
4321
4322 sub vmconfig_delete_or_detach_drive {
4323     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4324
4325     my $drive = parse_drive($opt, $conf->{$opt});
4326
4327     my $rpcenv = PVE::RPCEnvironment::get();
4328     my $authuser = $rpcenv->get_user();
4329
4330     if ($force) {
4331         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4332         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4333     } else {
4334         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4335     }
4336 }
4337
4338 sub vmconfig_apply_pending {
4339     my ($vmid, $conf, $storecfg) = @_;
4340
4341     # cold plug
4342
4343     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4344     while (my ($opt, $force) = each %$pending_delete_hash) {
4345         die "internal error" if $opt =~ m/^unused/;
4346         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4347         if (!defined($conf->{$opt})) {
4348             vmconfig_undelete_pending_option($conf, $opt);
4349             PVE::QemuConfig->write_config($vmid, $conf);
4350         } elsif (is_valid_drivename($opt)) {
4351             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4352             vmconfig_undelete_pending_option($conf, $opt);
4353             delete $conf->{$opt};
4354             PVE::QemuConfig->write_config($vmid, $conf);
4355         } else {
4356             vmconfig_undelete_pending_option($conf, $opt);
4357             delete $conf->{$opt};
4358             PVE::QemuConfig->write_config($vmid, $conf);
4359         }
4360     }
4361
4362     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4363
4364     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4365         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4366
4367         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4368             # skip if nothing changed
4369         } elsif (is_valid_drivename($opt)) {
4370             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4371                 if defined($conf->{$opt});
4372             $conf->{$opt} = $conf->{pending}->{$opt};
4373         } else {
4374             $conf->{$opt} = $conf->{pending}->{$opt};
4375         }
4376
4377         delete $conf->{pending}->{$opt};
4378         PVE::QemuConfig->write_config($vmid, $conf);
4379     }
4380 }
4381
4382 my $safe_num_ne = sub {
4383     my ($a, $b) = @_;
4384
4385     return 0 if !defined($a) && !defined($b);
4386     return 1 if !defined($a);
4387     return 1 if !defined($b);
4388
4389     return $a != $b;
4390 };
4391
4392 my $safe_string_ne = sub {
4393     my ($a, $b) = @_;
4394
4395     return 0 if !defined($a) && !defined($b);
4396     return 1 if !defined($a);
4397     return 1 if !defined($b);
4398
4399     return $a ne $b;
4400 };
4401
4402 sub vmconfig_update_net {
4403     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4404
4405     my $newnet = parse_net($value);
4406
4407     if ($conf->{$opt}) {
4408         my $oldnet = parse_net($conf->{$opt});
4409
4410         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4411             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4412             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4413             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4414
4415             # for non online change, we try to hot-unplug
4416             die "skip\n" if !$hotplug;
4417             vm_deviceunplug($vmid, $conf, $opt);
4418         } else {
4419
4420             die "internal error" if $opt !~ m/net(\d+)/;
4421             my $iface = "tap${vmid}i$1";
4422
4423             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4424                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4425                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4426                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4427                 PVE::Network::tap_unplug($iface);
4428                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4429             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4430                 # Rate can be applied on its own but any change above needs to
4431                 # include the rate in tap_plug since OVS resets everything.
4432                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4433             }
4434
4435             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4436                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4437             }
4438
4439             return 1;
4440         }
4441     }
4442
4443     if ($hotplug) {
4444         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4445     } else {
4446         die "skip\n";
4447     }
4448 }
4449
4450 sub vmconfig_update_disk {
4451     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4452
4453     # fixme: do we need force?
4454
4455     my $drive = parse_drive($opt, $value);
4456
4457     if ($conf->{$opt}) {
4458
4459         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4460
4461             my $media = $drive->{media} || 'disk';
4462             my $oldmedia = $old_drive->{media} || 'disk';
4463             die "unable to change media type\n" if $media ne $oldmedia;
4464
4465             if (!drive_is_cdrom($old_drive)) {
4466
4467                 if ($drive->{file} ne $old_drive->{file}) {
4468
4469                     die "skip\n" if !$hotplug;
4470
4471                     # unplug and register as unused
4472                     vm_deviceunplug($vmid, $conf, $opt);
4473                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4474
4475                 } else {
4476                     # update existing disk
4477
4478                     # skip non hotpluggable value
4479                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4480                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4481                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4482                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4483                         die "skip\n";
4484                     }
4485
4486                     # apply throttle
4487                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4488                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4489                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4490                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4491                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4492                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4493                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4494                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4495                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4496                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4497                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4498                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4499                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4500                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4501                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4502                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4503                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4504                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4505
4506                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4507                                                    ($drive->{mbps} || 0)*1024*1024,
4508                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4509                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4510                                                    $drive->{iops} || 0,
4511                                                    $drive->{iops_rd} || 0,
4512                                                    $drive->{iops_wr} || 0,
4513                                                    ($drive->{mbps_max} || 0)*1024*1024,
4514                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4515                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4516                                                    $drive->{iops_max} || 0,
4517                                                    $drive->{iops_rd_max} || 0,
4518                                                    $drive->{iops_wr_max} || 0,
4519                                                    $drive->{bps_max_length} || 1,
4520                                                    $drive->{bps_rd_max_length} || 1,
4521                                                    $drive->{bps_wr_max_length} || 1,
4522                                                    $drive->{iops_max_length} || 1,
4523                                                    $drive->{iops_rd_max_length} || 1,
4524                                                    $drive->{iops_wr_max_length} || 1);
4525
4526                     }
4527
4528                     return 1;
4529                 }
4530
4531             } else { # cdrom
4532
4533                 if ($drive->{file} eq 'none') {
4534                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4535                 } else {
4536                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4537                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4538                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4539                 }
4540
4541                 return 1;
4542             }
4543         }
4544     }
4545
4546     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4547     # hotplug new disks
4548     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4549     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4550 }
4551
4552 sub vm_start {
4553     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4554         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4555
4556     PVE::QemuConfig->lock_config($vmid, sub {
4557         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4558
4559         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4560
4561         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4562
4563         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4564
4565         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4566             vmconfig_apply_pending($vmid, $conf, $storecfg);
4567             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4568         }
4569
4570         my $defaults = load_defaults();
4571
4572         # set environment variable useful inside network script
4573         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4574
4575         my $local_volumes = {};
4576
4577         if ($targetstorage) {
4578             foreach_drive($conf, sub {
4579                 my ($ds, $drive) = @_;
4580
4581                 return if drive_is_cdrom($drive);
4582
4583                 my $volid = $drive->{file};
4584
4585                 return if !$volid;
4586
4587                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4588
4589                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4590                 return if $scfg->{shared};
4591                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4592             });
4593
4594             my $format = undef;
4595
4596             foreach my $opt (sort keys %$local_volumes) {
4597
4598                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4599                 my $drive = parse_drive($opt, $conf->{$opt});
4600
4601                 #if remote storage is specified, use default format
4602                 if ($targetstorage && $targetstorage ne "1") {
4603                     $storeid = $targetstorage;
4604                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4605                     $format = $defFormat;
4606                 } else {
4607                     #else we use same format than original
4608                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4609                     $format = qemu_img_format($scfg, $volid);
4610                 }
4611
4612                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4613                 my $newdrive = $drive;
4614                 $newdrive->{format} = $format;
4615                 $newdrive->{file} = $newvolid;
4616                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4617                 $local_volumes->{$opt} = $drivestr;
4618                 #pass drive to conf for command line
4619                 $conf->{$opt} = $drivestr;
4620             }
4621         }
4622
4623         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4624
4625         my $migrate_port = 0;
4626         my $migrate_uri;
4627         if ($statefile) {
4628             if ($statefile eq 'tcp') {
4629                 my $localip = "localhost";
4630                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4631                 my $nodename = PVE::INotify::nodename();
4632
4633                 if (!defined($migration_type)) {
4634                     if (defined($datacenterconf->{migration}->{type})) {
4635                         $migration_type = $datacenterconf->{migration}->{type};
4636                     } else {
4637                         $migration_type = 'secure';
4638                     }
4639                 }
4640
4641                 if ($migration_type eq 'insecure') {
4642                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4643                     if ($migrate_network_addr) {
4644                         $localip = $migrate_network_addr;
4645                     } else {
4646                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4647                     }
4648
4649                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4650                 }
4651
4652                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4653                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4654                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4655                 push @$cmd, '-incoming', $migrate_uri;
4656                 push @$cmd, '-S';
4657
4658             } elsif ($statefile eq 'unix') {
4659                 # should be default for secure migrations as a ssh TCP forward
4660                 # tunnel is not deterministic reliable ready and fails regurarly
4661                 # to set up in time, so use UNIX socket forwards
4662                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4663                 unlink $socket_addr;
4664
4665                 $migrate_uri = "unix:$socket_addr";
4666
4667                 push @$cmd, '-incoming', $migrate_uri;
4668                 push @$cmd, '-S';
4669
4670             } else {
4671                 push @$cmd, '-loadstate', $statefile;
4672             }
4673         } elsif ($paused) {
4674             push @$cmd, '-S';
4675         }
4676
4677         # host pci devices
4678         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4679           my $d = parse_hostpci($conf->{"hostpci$i"});
4680           next if !$d;
4681           my $pcidevices = $d->{pciid};
4682           foreach my $pcidevice (@$pcidevices) {
4683                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4684
4685                 my $info = pci_device_info("0000:$pciid");
4686                 die "IOMMU not present\n" if !check_iommu_support();
4687                 die "no pci device info for device '$pciid'\n" if !$info;
4688                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4689                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4690           }
4691         }
4692
4693         PVE::Storage::activate_volumes($storecfg, $vollist);
4694
4695         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4696             my $cmd = [];
4697             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4698             eval  { run_command($cmd); };
4699         }
4700
4701         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4702                                                   : $defaults->{cpuunits};
4703
4704         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4705         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4706
4707         my %properties = (
4708             Slice => 'qemu.slice',
4709             KillMode => 'none',
4710             CPUShares => $cpuunits
4711         );
4712
4713         if (my $cpulimit = $conf->{cpulimit}) {
4714             $properties{CPUQuota} = int($cpulimit * 100);
4715         }
4716         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4717
4718         if ($conf->{hugepages}) {
4719
4720             my $code = sub {
4721                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4722                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4723
4724                 PVE::QemuServer::Memory::hugepages_mount();
4725                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4726
4727                 eval  {
4728                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4729                     run_command($cmd, %run_params);
4730                 };
4731
4732                 if (my $err = $@) {
4733                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4734                     die $err;
4735                 }
4736
4737                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4738             };
4739             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4740
4741         } else {
4742             eval  {
4743                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4744                 run_command($cmd, %run_params);
4745             };
4746         }
4747
4748         if (my $err = $@) {
4749             # deactivate volumes if start fails
4750             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4751             die "start failed: $err";
4752         }
4753
4754         print "migration listens on $migrate_uri\n" if $migrate_uri;
4755
4756         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4757             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4758             warn $@ if $@;
4759         }
4760
4761         #start nbd server for storage migration
4762         if ($targetstorage) {
4763             my $nodename = PVE::INotify::nodename();
4764             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4765             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4766             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4767             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4768
4769             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4770
4771             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4772
4773             foreach my $opt (sort keys %$local_volumes) {
4774                 my $volid = $local_volumes->{$opt};
4775                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4776                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4777                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4778             }
4779         }
4780
4781         if ($migratedfrom) {
4782             eval {
4783                 set_migration_caps($vmid);
4784             };
4785             warn $@ if $@;
4786
4787             if ($spice_port) {
4788                 print "spice listens on port $spice_port\n";
4789                 if ($spice_ticket) {
4790                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4791                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4792                 }
4793             }
4794
4795         } else {
4796             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4797                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4798                     if $conf->{balloon};
4799             }
4800
4801             foreach my $opt (keys %$conf) {
4802                 next if $opt !~  m/^net\d+$/;
4803                 my $nicconf = parse_net($conf->{$opt});
4804                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4805             }
4806         }
4807
4808         vm_mon_cmd_nocheck($vmid, 'qom-set',
4809                     path => "machine/peripheral/balloon0",
4810                     property => "guest-stats-polling-interval",
4811                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4812
4813     });
4814 }
4815
4816 sub vm_mon_cmd {
4817     my ($vmid, $execute, %params) = @_;
4818
4819     my $cmd = { execute => $execute, arguments => \%params };
4820     vm_qmp_command($vmid, $cmd);
4821 }
4822
4823 sub vm_mon_cmd_nocheck {
4824     my ($vmid, $execute, %params) = @_;
4825
4826     my $cmd = { execute => $execute, arguments => \%params };
4827     vm_qmp_command($vmid, $cmd, 1);
4828 }
4829
4830 sub vm_qmp_command {
4831     my ($vmid, $cmd, $nocheck) = @_;
4832
4833     my $res;
4834
4835     my $timeout;
4836     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4837         $timeout = $cmd->{arguments}->{timeout};
4838         delete $cmd->{arguments}->{timeout};
4839     }
4840
4841     eval {
4842         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4843         my $sname = qmp_socket($vmid);
4844         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4845             my $qmpclient = PVE::QMPClient->new();
4846
4847             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4848         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4849             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4850                 if scalar(%{$cmd->{arguments}});
4851             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4852         } else {
4853             die "unable to open monitor socket\n";
4854         }
4855     };
4856     if (my $err = $@) {
4857         syslog("err", "VM $vmid qmp command failed - $err");
4858         die $err;
4859     }
4860
4861     return $res;
4862 }
4863
4864 sub vm_human_monitor_command {
4865     my ($vmid, $cmdline) = @_;
4866
4867     my $res;
4868
4869     my $cmd = {
4870         execute => 'human-monitor-command',
4871         arguments => { 'command-line' => $cmdline},
4872     };
4873
4874     return vm_qmp_command($vmid, $cmd);
4875 }
4876
4877 sub vm_commandline {
4878     my ($storecfg, $vmid) = @_;
4879
4880     my $conf = PVE::QemuConfig->load_config($vmid);
4881
4882     my $defaults = load_defaults();
4883
4884     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4885
4886     return PVE::Tools::cmd2string($cmd);
4887 }
4888
4889 sub vm_reset {
4890     my ($vmid, $skiplock) = @_;
4891
4892     PVE::QemuConfig->lock_config($vmid, sub {
4893
4894         my $conf = PVE::QemuConfig->load_config($vmid);
4895
4896         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4897
4898         vm_mon_cmd($vmid, "system_reset");
4899     });
4900 }
4901
4902 sub get_vm_volumes {
4903     my ($conf) = @_;
4904
4905     my $vollist = [];
4906     foreach_volid($conf, sub {
4907         my ($volid, $attr) = @_;
4908
4909         return if $volid =~ m|^/|;
4910
4911         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4912         return if !$sid;
4913
4914         push @$vollist, $volid;
4915     });
4916
4917     return $vollist;
4918 }
4919
4920 sub vm_stop_cleanup {
4921     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4922
4923     eval {
4924
4925         if (!$keepActive) {
4926             my $vollist = get_vm_volumes($conf);
4927             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4928         }
4929
4930         foreach my $ext (qw(mon qmp pid vnc qga)) {
4931             unlink "/var/run/qemu-server/${vmid}.$ext";
4932         }
4933
4934         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4935     };
4936     warn $@ if $@; # avoid errors - just warn
4937 }
4938
4939 # Note: use $nockeck to skip tests if VM configuration file exists.
4940 # We need that when migration VMs to other nodes (files already moved)
4941 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4942 sub vm_stop {
4943     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4944
4945     $force = 1 if !defined($force) && !$shutdown;
4946
4947     if ($migratedfrom){
4948         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4949         kill 15, $pid if $pid;
4950         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4951         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4952         return;
4953     }
4954
4955     PVE::QemuConfig->lock_config($vmid, sub {
4956
4957         my $pid = check_running($vmid, $nocheck);
4958         return if !$pid;
4959
4960         my $conf;
4961         if (!$nocheck) {
4962             $conf = PVE::QemuConfig->load_config($vmid);
4963             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4964             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4965                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4966                 $timeout = $opts->{down} if $opts->{down};
4967             }
4968         }
4969
4970         $timeout = 60 if !defined($timeout);
4971
4972         eval {
4973             if ($shutdown) {
4974                 if (defined($conf) && $conf->{agent}) {
4975                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4976                 } else {
4977                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4978                 }
4979             } else {
4980                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4981             }
4982         };
4983         my $err = $@;
4984
4985         if (!$err) {
4986             my $count = 0;
4987             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4988                 $count++;
4989                 sleep 1;
4990             }
4991
4992             if ($count >= $timeout) {
4993                 if ($force) {
4994                     warn "VM still running - terminating now with SIGTERM\n";
4995                     kill 15, $pid;
4996                 } else {
4997                     die "VM quit/powerdown failed - got timeout\n";
4998                 }
4999             } else {
5000                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5001                 return;
5002             }
5003         } else {
5004             if ($force) {
5005                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5006                 kill 15, $pid;
5007             } else {
5008                 die "VM quit/powerdown failed\n";
5009             }
5010         }
5011
5012         # wait again
5013         $timeout = 10;
5014
5015         my $count = 0;
5016         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5017             $count++;
5018             sleep 1;
5019         }
5020
5021         if ($count >= $timeout) {
5022             warn "VM still running - terminating now with SIGKILL\n";
5023             kill 9, $pid;
5024             sleep 1;
5025         }
5026
5027         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5028    });
5029 }
5030
5031 sub vm_suspend {
5032     my ($vmid, $skiplock) = @_;
5033
5034     PVE::QemuConfig->lock_config($vmid, sub {
5035
5036         my $conf = PVE::QemuConfig->load_config($vmid);
5037
5038         PVE::QemuConfig->check_lock($conf)
5039             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5040
5041         vm_mon_cmd($vmid, "stop");
5042     });
5043 }
5044
5045 sub vm_resume {
5046     my ($vmid, $skiplock, $nocheck) = @_;
5047
5048     PVE::QemuConfig->lock_config($vmid, sub {
5049
5050         if (!$nocheck) {
5051
5052             my $conf = PVE::QemuConfig->load_config($vmid);
5053
5054             PVE::QemuConfig->check_lock($conf)
5055                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5056
5057             vm_mon_cmd($vmid, "cont");
5058
5059         } else {
5060             vm_mon_cmd_nocheck($vmid, "cont");
5061         }
5062     });
5063 }
5064
5065 sub vm_sendkey {
5066     my ($vmid, $skiplock, $key) = @_;
5067
5068     PVE::QemuConfig->lock_config($vmid, sub {
5069
5070         my $conf = PVE::QemuConfig->load_config($vmid);
5071
5072         # there is no qmp command, so we use the human monitor command
5073         vm_human_monitor_command($vmid, "sendkey $key");
5074     });
5075 }
5076
5077 sub vm_destroy {
5078     my ($storecfg, $vmid, $skiplock) = @_;
5079
5080     PVE::QemuConfig->lock_config($vmid, sub {
5081
5082         my $conf = PVE::QemuConfig->load_config($vmid);
5083
5084         if (!check_running($vmid)) {
5085             destroy_vm($storecfg, $vmid, undef, $skiplock);
5086         } else {
5087             die "VM $vmid is running - destroy failed\n";
5088         }
5089     });
5090 }
5091
5092 # pci helpers
5093
5094 sub file_write {
5095     my ($filename, $buf) = @_;
5096
5097     my $fh = IO::File->new($filename, "w");
5098     return undef if !$fh;
5099
5100     my $res = print $fh $buf;
5101
5102     $fh->close();
5103
5104     return $res;
5105 }
5106
5107 sub pci_device_info {
5108     my ($name) = @_;
5109
5110     my $res;
5111
5112     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5113     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5114
5115     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5116     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5117
5118     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5119     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5120
5121     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5122     return undef if !defined($product) || $product !~ s/^0x//;
5123
5124     $res = {
5125         name => $name,
5126         vendor => $vendor,
5127         product => $product,
5128         domain => $domain,
5129         bus => $bus,
5130         slot => $slot,
5131         func => $func,
5132         irq => $irq,
5133         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5134     };
5135
5136     return $res;
5137 }
5138
5139 sub pci_dev_reset {
5140     my ($dev) = @_;
5141
5142     my $name = $dev->{name};
5143
5144     my $fn = "$pcisysfs/devices/$name/reset";
5145
5146     return file_write($fn, "1");
5147 }
5148
5149 sub pci_dev_bind_to_vfio {
5150     my ($dev) = @_;
5151
5152     my $name = $dev->{name};
5153
5154     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5155
5156     if (!-d $vfio_basedir) {
5157         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5158     }
5159     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5160
5161     my $testdir = "$vfio_basedir/$name";
5162     return 1 if -d $testdir;
5163
5164     my $data = "$dev->{vendor} $dev->{product}";
5165     return undef if !file_write("$vfio_basedir/new_id", $data);
5166
5167     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5168     if (!file_write($fn, $name)) {
5169         return undef if -f $fn;
5170     }
5171
5172     $fn = "$vfio_basedir/bind";
5173     if (! -d $testdir) {
5174         return undef if !file_write($fn, $name);
5175     }
5176
5177     return -d $testdir;
5178 }
5179
5180 sub pci_dev_group_bind_to_vfio {
5181     my ($pciid) = @_;
5182
5183     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5184
5185     if (!-d $vfio_basedir) {
5186         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5187     }
5188     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5189
5190     # get IOMMU group devices
5191     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5192       my @devs = grep /^0000:/, readdir($D);
5193     closedir($D);
5194
5195     foreach my $pciid (@devs) {
5196         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5197
5198         # pci bridges, switches or root ports are not supported
5199         # they have a pci_bus subdirectory so skip them
5200         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5201
5202         my $info = pci_device_info($1);
5203         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5204     }
5205
5206     return 1;
5207 }
5208
5209 # vzdump restore implementaion
5210
5211 sub tar_archive_read_firstfile {
5212     my $archive = shift;
5213
5214     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5215
5216     # try to detect archive type first
5217     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5218         die "unable to open file '$archive'\n";
5219     my $firstfile = <$fh>;
5220     kill 15, $pid;
5221     close $fh;
5222
5223     die "ERROR: archive contaions no data\n" if !$firstfile;
5224     chomp $firstfile;
5225
5226     return $firstfile;
5227 }
5228
5229 sub tar_restore_cleanup {
5230     my ($storecfg, $statfile) = @_;
5231
5232     print STDERR "starting cleanup\n";
5233
5234     if (my $fd = IO::File->new($statfile, "r")) {
5235         while (defined(my $line = <$fd>)) {
5236             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5237                 my $volid = $2;
5238                 eval {
5239                     if ($volid =~ m|^/|) {
5240                         unlink $volid || die 'unlink failed\n';
5241                     } else {
5242                         PVE::Storage::vdisk_free($storecfg, $volid);
5243                     }
5244                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5245                 };
5246                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5247             } else {
5248                 print STDERR "unable to parse line in statfile - $line";
5249             }
5250         }
5251         $fd->close();
5252     }
5253 }
5254
5255 sub restore_archive {
5256     my ($archive, $vmid, $user, $opts) = @_;
5257
5258     my $format = $opts->{format};
5259     my $comp;
5260
5261     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5262         $format = 'tar' if !$format;
5263         $comp = 'gzip';
5264     } elsif ($archive =~ m/\.tar$/) {
5265         $format = 'tar' if !$format;
5266     } elsif ($archive =~ m/.tar.lzo$/) {
5267         $format = 'tar' if !$format;
5268         $comp = 'lzop';
5269     } elsif ($archive =~ m/\.vma$/) {
5270         $format = 'vma' if !$format;
5271     } elsif ($archive =~ m/\.vma\.gz$/) {
5272         $format = 'vma' if !$format;
5273         $comp = 'gzip';
5274     } elsif ($archive =~ m/\.vma\.lzo$/) {
5275         $format = 'vma' if !$format;
5276         $comp = 'lzop';
5277     } else {
5278         $format = 'vma' if !$format; # default
5279     }
5280
5281     # try to detect archive format
5282     if ($format eq 'tar') {
5283         return restore_tar_archive($archive, $vmid, $user, $opts);
5284     } else {
5285         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5286     }
5287 }
5288
5289 sub restore_update_config_line {
5290     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5291
5292     return if $line =~ m/^\#qmdump\#/;
5293     return if $line =~ m/^\#vzdump\#/;
5294     return if $line =~ m/^lock:/;
5295     return if $line =~ m/^unused\d+:/;
5296     return if $line =~ m/^parent:/;
5297     return if $line =~ m/^template:/; # restored VM is never a template
5298
5299     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5300     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5301         # try to convert old 1.X settings
5302         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5303         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5304             my ($model, $macaddr) = split(/\=/, $devconfig);
5305             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5306             my $net = {
5307                 model => $model,
5308                 bridge => "vmbr$ind",
5309                 macaddr => $macaddr,
5310             };
5311             my $netstr = print_net($net);
5312
5313             print $outfd "net$cookie->{netcount}: $netstr\n";
5314             $cookie->{netcount}++;
5315         }
5316     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5317         my ($id, $netstr) = ($1, $2);
5318         my $net = parse_net($netstr);
5319         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5320         $netstr = print_net($net);
5321         print $outfd "$id: $netstr\n";
5322     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5323         my $virtdev = $1;
5324         my $value = $3;
5325         my $di = parse_drive($virtdev, $value);
5326         if (defined($di->{backup}) && !$di->{backup}) {
5327             print $outfd "#$line";
5328         } elsif ($map->{$virtdev}) {
5329             delete $di->{format}; # format can change on restore
5330             $di->{file} = $map->{$virtdev};
5331             $value = print_drive($vmid, $di);
5332             print $outfd "$virtdev: $value\n";
5333         } else {
5334             print $outfd $line;
5335         }
5336     } else {
5337         print $outfd $line;
5338     }
5339 }
5340
5341 sub scan_volids {
5342     my ($cfg, $vmid) = @_;
5343
5344     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5345
5346     my $volid_hash = {};
5347     foreach my $storeid (keys %$info) {
5348         foreach my $item (@{$info->{$storeid}}) {
5349             next if !($item->{volid} && $item->{size});
5350             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5351             $volid_hash->{$item->{volid}} = $item;
5352         }
5353     }
5354
5355     return $volid_hash;
5356 }
5357
5358 sub is_volume_in_use {
5359     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5360
5361     my $path = PVE::Storage::path($storecfg, $volid);
5362
5363     my $scan_config = sub {
5364         my ($cref, $snapname) = @_;
5365
5366         foreach my $key (keys %$cref) {
5367             my $value = $cref->{$key};
5368             if (is_valid_drivename($key)) {
5369                 next if $skip_drive && $key eq $skip_drive;
5370                 my $drive = parse_drive($key, $value);
5371                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5372                 return 1 if $volid eq $drive->{file};
5373                 if ($drive->{file} =~ m!^/!) {
5374                     return 1 if $drive->{file} eq $path;
5375                 } else {
5376                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5377                     next if !$storeid;
5378                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5379                     next if !$scfg;
5380                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5381                 }
5382             }
5383         }
5384
5385         return 0;
5386     };
5387
5388     return 1 if &$scan_config($conf);
5389
5390     undef $skip_drive;
5391
5392     foreach my $snapname (keys %{$conf->{snapshots}}) {
5393         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5394     }
5395
5396     return 0;
5397 }
5398
5399 sub update_disksize {
5400     my ($vmid, $conf, $volid_hash) = @_;
5401
5402     my $changes;
5403
5404     # used and unused disks
5405     my $referenced = {};
5406
5407     # Note: it is allowed to define multiple storages with same path (alias), so
5408     # we need to check both 'volid' and real 'path' (two different volid can point
5409     # to the same path).
5410
5411     my $referencedpath = {};
5412
5413     # update size info
5414     foreach my $opt (keys %$conf) {
5415         if (is_valid_drivename($opt)) {
5416             my $drive = parse_drive($opt, $conf->{$opt});
5417             my $volid = $drive->{file};
5418             next if !$volid;
5419
5420             $referenced->{$volid} = 1;
5421             if ($volid_hash->{$volid} &&
5422                 (my $path = $volid_hash->{$volid}->{path})) {
5423                 $referencedpath->{$path} = 1;
5424             }
5425
5426             next if drive_is_cdrom($drive);
5427             next if !$volid_hash->{$volid};
5428
5429             $drive->{size} = $volid_hash->{$volid}->{size};
5430             my $new = print_drive($vmid, $drive);
5431             if ($new ne $conf->{$opt}) {
5432                 $changes = 1;
5433                 $conf->{$opt} = $new;
5434             }
5435         }
5436     }
5437
5438     # remove 'unusedX' entry if volume is used
5439     foreach my $opt (keys %$conf) {
5440         next if $opt !~ m/^unused\d+$/;
5441         my $volid = $conf->{$opt};
5442         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5443         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5444             $changes = 1;
5445             delete $conf->{$opt};
5446         }
5447
5448         $referenced->{$volid} = 1;
5449         $referencedpath->{$path} = 1 if $path;
5450     }
5451
5452     foreach my $volid (sort keys %$volid_hash) {
5453         next if $volid =~ m/vm-$vmid-state-/;
5454         next if $referenced->{$volid};
5455         my $path = $volid_hash->{$volid}->{path};
5456         next if !$path; # just to be sure
5457         next if $referencedpath->{$path};
5458         $changes = 1;
5459         PVE::QemuConfig->add_unused_volume($conf, $volid);
5460         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5461     }
5462
5463     return $changes;
5464 }
5465
5466 sub rescan {
5467     my ($vmid, $nolock) = @_;
5468
5469     my $cfg = PVE::Storage::config();
5470
5471     my $volid_hash = scan_volids($cfg, $vmid);
5472
5473     my $updatefn =  sub {
5474         my ($vmid) = @_;
5475
5476         my $conf = PVE::QemuConfig->load_config($vmid);
5477
5478         PVE::QemuConfig->check_lock($conf);
5479
5480         my $vm_volids = {};
5481         foreach my $volid (keys %$volid_hash) {
5482             my $info = $volid_hash->{$volid};
5483             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5484         }
5485
5486         my $changes = update_disksize($vmid, $conf, $vm_volids);
5487
5488         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5489     };
5490
5491     if (defined($vmid)) {
5492         if ($nolock) {
5493             &$updatefn($vmid);
5494         } else {
5495             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5496         }
5497     } else {
5498         my $vmlist = config_list();
5499         foreach my $vmid (keys %$vmlist) {
5500             if ($nolock) {
5501                 &$updatefn($vmid);
5502             } else {
5503                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5504             }
5505         }
5506     }
5507 }
5508
5509 sub restore_vma_archive {
5510     my ($archive, $vmid, $user, $opts, $comp) = @_;
5511
5512     my $input = $archive eq '-' ? "<&STDIN" : undef;
5513     my $readfrom = $archive;
5514
5515     my $uncomp = '';
5516     if ($comp) {
5517         $readfrom = '-';
5518         my $qarchive = PVE::Tools::shellquote($archive);
5519         if ($comp eq 'gzip') {
5520             $uncomp = "zcat $qarchive|";
5521         } elsif ($comp eq 'lzop') {
5522             $uncomp = "lzop -d -c $qarchive|";
5523         } else {
5524             die "unknown compression method '$comp'\n";
5525         }
5526
5527     }
5528
5529     my $tmpdir = "/var/tmp/vzdumptmp$$";
5530     rmtree $tmpdir;
5531
5532     # disable interrupts (always do cleanups)
5533     local $SIG{INT} =
5534         local $SIG{TERM} =
5535         local $SIG{QUIT} =
5536         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5537
5538     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5539     POSIX::mkfifo($mapfifo, 0600);
5540     my $fifofh;
5541
5542     my $openfifo = sub {
5543         open($fifofh, '>', $mapfifo) || die $!;
5544     };
5545
5546     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5547
5548     my $oldtimeout;
5549     my $timeout = 5;
5550
5551     my $devinfo = {};
5552
5553     my $rpcenv = PVE::RPCEnvironment::get();
5554
5555     my $conffile = PVE::QemuConfig->config_file($vmid);
5556     my $tmpfn = "$conffile.$$.tmp";
5557
5558     # Note: $oldconf is undef if VM does not exists
5559     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5560     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5561
5562     my $print_devmap = sub {
5563         my $virtdev_hash = {};
5564
5565         my $cfgfn = "$tmpdir/qemu-server.conf";
5566
5567         # we can read the config - that is already extracted
5568         my $fh = IO::File->new($cfgfn, "r") ||
5569             "unable to read qemu-server.conf - $!\n";
5570
5571         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5572         if (-f $fwcfgfn) {
5573             my $pve_firewall_dir = '/etc/pve/firewall';
5574             mkdir $pve_firewall_dir; # make sure the dir exists
5575             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5576         }
5577
5578         while (defined(my $line = <$fh>)) {
5579             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5580                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5581                 die "archive does not contain data for drive '$virtdev'\n"
5582                     if !$devinfo->{$devname};
5583                 if (defined($opts->{storage})) {
5584                     $storeid = $opts->{storage} || 'local';
5585                 } elsif (!$storeid) {
5586                     $storeid = 'local';
5587                 }
5588                 $format = 'raw' if !$format;
5589                 $devinfo->{$devname}->{devname} = $devname;
5590                 $devinfo->{$devname}->{virtdev} = $virtdev;
5591                 $devinfo->{$devname}->{format} = $format;
5592                 $devinfo->{$devname}->{storeid} = $storeid;
5593
5594                 # check permission on storage
5595                 my $pool = $opts->{pool}; # todo: do we need that?
5596                 if ($user ne 'root@pam') {
5597                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5598                 }
5599
5600                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5601             }
5602         }
5603
5604         foreach my $devname (keys %$devinfo) {
5605             die "found no device mapping information for device '$devname'\n"
5606                 if !$devinfo->{$devname}->{virtdev};
5607         }
5608
5609         my $cfg = PVE::Storage::config();
5610
5611         # create empty/temp config
5612         if ($oldconf) {
5613             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5614             foreach_drive($oldconf, sub {
5615                 my ($ds, $drive) = @_;
5616
5617                 return if drive_is_cdrom($drive);
5618
5619                 my $volid = $drive->{file};
5620
5621                 return if !$volid || $volid =~ m|^/|;
5622
5623                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5624                 return if !$path || !$owner || ($owner != $vmid);
5625
5626                 # Note: only delete disk we want to restore
5627                 # other volumes will become unused
5628                 if ($virtdev_hash->{$ds}) {
5629                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5630                     if (my $err = $@) {
5631                         warn $err;
5632                     }
5633                 }
5634             });
5635
5636             # delete vmstate files
5637             # since after the restore we have no snapshots anymore
5638             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5639                 my $snap = $oldconf->{snapshots}->{$snapname};
5640                 if ($snap->{vmstate}) {
5641                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5642                     if (my $err = $@) {
5643                         warn $err;
5644                     }
5645                 }
5646             }
5647         }
5648
5649         my $map = {};
5650         foreach my $virtdev (sort keys %$virtdev_hash) {
5651             my $d = $virtdev_hash->{$virtdev};
5652             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5653             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5654
5655             # test if requested format is supported
5656             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5657             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5658             $d->{format} = $defFormat if !$supported;
5659
5660             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5661                                                   $d->{format}, undef, $alloc_size);
5662             print STDERR "new volume ID is '$volid'\n";
5663             $d->{volid} = $volid;
5664             my $path = PVE::Storage::path($cfg, $volid);
5665
5666             PVE::Storage::activate_volumes($cfg,[$volid]);
5667
5668             my $write_zeros = 1;
5669             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5670                 $write_zeros = 0;
5671             }
5672
5673             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5674
5675             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5676             $map->{$virtdev} = $volid;
5677         }
5678
5679         $fh->seek(0, 0) || die "seek failed - $!\n";
5680
5681         my $outfd = new IO::File ($tmpfn, "w") ||
5682             die "unable to write config for VM $vmid\n";
5683
5684         my $cookie = { netcount => 0 };
5685         while (defined(my $line = <$fh>)) {
5686             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5687         }
5688
5689         $fh->close();
5690         $outfd->close();
5691     };
5692
5693     eval {
5694         # enable interrupts
5695         local $SIG{INT} =
5696             local $SIG{TERM} =
5697             local $SIG{QUIT} =
5698             local $SIG{HUP} =
5699             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5700         local $SIG{ALRM} = sub { die "got timeout\n"; };
5701
5702         $oldtimeout = alarm($timeout);
5703
5704         my $parser = sub {
5705             my $line = shift;
5706
5707             print "$line\n";
5708
5709             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5710                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5711                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5712             } elsif ($line =~ m/^CTIME: /) {
5713                 # we correctly received the vma config, so we can disable
5714                 # the timeout now for disk allocation (set to 10 minutes, so
5715                 # that we always timeout if something goes wrong)
5716                 alarm(600);
5717                 &$print_devmap();
5718                 print $fifofh "done\n";
5719                 my $tmp = $oldtimeout || 0;
5720                 $oldtimeout = undef;
5721                 alarm($tmp);
5722                 close($fifofh);
5723             }
5724         };
5725
5726         print "restore vma archive: $cmd\n";
5727         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5728     };
5729     my $err = $@;
5730
5731     alarm($oldtimeout) if $oldtimeout;
5732
5733     my $vollist = [];
5734     foreach my $devname (keys %$devinfo) {
5735         my $volid = $devinfo->{$devname}->{volid};
5736         push @$vollist, $volid if $volid;
5737     }
5738
5739     my $cfg = PVE::Storage::config();
5740     PVE::Storage::deactivate_volumes($cfg, $vollist);
5741
5742     unlink $mapfifo;
5743
5744     if ($err) {
5745         rmtree $tmpdir;
5746         unlink $tmpfn;
5747
5748         foreach my $devname (keys %$devinfo) {
5749             my $volid = $devinfo->{$devname}->{volid};
5750             next if !$volid;
5751             eval {
5752                 if ($volid =~ m|^/|) {
5753                     unlink $volid || die 'unlink failed\n';
5754                 } else {
5755                     PVE::Storage::vdisk_free($cfg, $volid);
5756                 }
5757                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5758             };
5759             print STDERR "unable to cleanup '$volid' - $@" if $@;
5760         }
5761         die $err;
5762     }
5763
5764     rmtree $tmpdir;
5765
5766     rename($tmpfn, $conffile) ||
5767         die "unable to commit configuration file '$conffile'\n";
5768
5769     PVE::Cluster::cfs_update(); # make sure we read new file
5770
5771     eval { rescan($vmid, 1); };
5772     warn $@ if $@;
5773 }
5774
5775 sub restore_tar_archive {
5776     my ($archive, $vmid, $user, $opts) = @_;
5777
5778     if ($archive ne '-') {
5779         my $firstfile = tar_archive_read_firstfile($archive);
5780         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5781             if $firstfile ne 'qemu-server.conf';
5782     }
5783
5784     my $storecfg = PVE::Storage::config();
5785
5786     # destroy existing data - keep empty config
5787     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5788     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5789
5790     my $tocmd = "/usr/lib/qemu-server/qmextract";
5791
5792     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5793     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5794     $tocmd .= ' --prealloc' if $opts->{prealloc};
5795     $tocmd .= ' --info' if $opts->{info};
5796
5797     # tar option "xf" does not autodetect compression when read from STDIN,
5798     # so we pipe to zcat
5799     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5800         PVE::Tools::shellquote("--to-command=$tocmd");
5801
5802     my $tmpdir = "/var/tmp/vzdumptmp$$";
5803     mkpath $tmpdir;
5804
5805     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5806     local $ENV{VZDUMP_VMID} = $vmid;
5807     local $ENV{VZDUMP_USER} = $user;
5808
5809     my $conffile = PVE::QemuConfig->config_file($vmid);
5810     my $tmpfn = "$conffile.$$.tmp";
5811
5812     # disable interrupts (always do cleanups)
5813     local $SIG{INT} =
5814         local $SIG{TERM} =
5815         local $SIG{QUIT} =
5816         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5817
5818     eval {
5819         # enable interrupts
5820         local $SIG{INT} =
5821             local $SIG{TERM} =
5822             local $SIG{QUIT} =
5823             local $SIG{HUP} =
5824             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5825
5826         if ($archive eq '-') {
5827             print "extracting archive from STDIN\n";
5828             run_command($cmd, input => "<&STDIN");
5829         } else {
5830             print "extracting archive '$archive'\n";
5831             run_command($cmd);
5832         }
5833
5834         return if $opts->{info};
5835
5836         # read new mapping
5837         my $map = {};
5838         my $statfile = "$tmpdir/qmrestore.stat";
5839         if (my $fd = IO::File->new($statfile, "r")) {
5840             while (defined (my $line = <$fd>)) {
5841                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5842                     $map->{$1} = $2 if $1;
5843                 } else {
5844                     print STDERR "unable to parse line in statfile - $line\n";
5845                 }
5846             }
5847             $fd->close();
5848         }
5849
5850         my $confsrc = "$tmpdir/qemu-server.conf";
5851
5852         my $srcfd = new IO::File($confsrc, "r") ||
5853             die "unable to open file '$confsrc'\n";
5854
5855         my $outfd = new IO::File ($tmpfn, "w") ||
5856             die "unable to write config for VM $vmid\n";
5857
5858         my $cookie = { netcount => 0 };
5859         while (defined (my $line = <$srcfd>)) {
5860             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5861         }
5862
5863         $srcfd->close();
5864         $outfd->close();
5865     };
5866     my $err = $@;
5867
5868     if ($err) {
5869
5870         unlink $tmpfn;
5871
5872         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5873
5874         die $err;
5875     }
5876
5877     rmtree $tmpdir;
5878
5879     rename $tmpfn, $conffile ||
5880         die "unable to commit configuration file '$conffile'\n";
5881
5882     PVE::Cluster::cfs_update(); # make sure we read new file
5883
5884     eval { rescan($vmid, 1); };
5885     warn $@ if $@;
5886 };
5887
5888 sub foreach_storage_used_by_vm {
5889     my ($conf, $func) = @_;
5890
5891     my $sidhash = {};
5892
5893     foreach_drive($conf, sub {
5894         my ($ds, $drive) = @_;
5895         return if drive_is_cdrom($drive);
5896
5897         my $volid = $drive->{file};
5898
5899         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5900         $sidhash->{$sid} = $sid if $sid;
5901     });
5902
5903     foreach my $sid (sort keys %$sidhash) {
5904         &$func($sid);
5905     }
5906 }
5907
5908 sub do_snapshots_with_qemu {
5909     my ($storecfg, $volid) = @_;
5910
5911     my $storage_name = PVE::Storage::parse_volume_id($volid);
5912
5913     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5914         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5915         return 1;
5916     }
5917
5918     if ($volid =~ m/\.(qcow2|qed)$/){
5919         return 1;
5920     }
5921
5922     return undef;
5923 }
5924
5925 sub qga_check_running {
5926     my ($vmid) = @_;
5927
5928     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5929     if ($@) {
5930         warn "Qemu Guest Agent is not running - $@";
5931         return 0;
5932     }
5933     return 1;
5934 }
5935
5936 sub template_create {
5937     my ($vmid, $conf, $disk) = @_;
5938
5939     my $storecfg = PVE::Storage::config();
5940
5941     foreach_drive($conf, sub {
5942         my ($ds, $drive) = @_;
5943
5944         return if drive_is_cdrom($drive);
5945         return if $disk && $ds ne $disk;
5946
5947         my $volid = $drive->{file};
5948         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5949
5950         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5951         $drive->{file} = $voliddst;
5952         $conf->{$ds} = print_drive($vmid, $drive);
5953         PVE::QemuConfig->write_config($vmid, $conf);
5954     });
5955 }
5956
5957 sub qemu_img_convert {
5958     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5959
5960     my $storecfg = PVE::Storage::config();
5961     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5962     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5963
5964     if ($src_storeid && $dst_storeid) {
5965
5966         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5967
5968         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5969         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5970
5971         my $src_format = qemu_img_format($src_scfg, $src_volname);
5972         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5973
5974         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5975         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5976
5977         my $cmd = [];
5978         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5979         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5980         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5981         if ($is_zero_initialized) {
5982             push @$cmd, "zeroinit:$dst_path";
5983         } else {
5984             push @$cmd, $dst_path;
5985         }
5986
5987         my $parser = sub {
5988             my $line = shift;
5989             if($line =~ m/\((\S+)\/100\%\)/){
5990                 my $percent = $1;
5991                 my $transferred = int($size * $percent / 100);
5992                 my $remaining = $size - $transferred;
5993
5994                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5995             }
5996
5997         };
5998
5999         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6000         my $err = $@;
6001         die "copy failed: $err" if $err;
6002     }
6003 }
6004
6005 sub qemu_img_format {
6006     my ($scfg, $volname) = @_;
6007
6008     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6009         return $1;
6010     } else {
6011         return "raw";
6012     }
6013 }
6014
6015 sub qemu_drive_mirror {
6016     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6017
6018     $jobs = {} if !$jobs;
6019
6020     my $qemu_target;
6021     my $format;
6022     $jobs->{"drive-$drive"} = {};
6023
6024     if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
6025         my $server = $1;
6026         my $port = $2;
6027         my $exportname = $3;
6028
6029         $format = "nbd";
6030         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
6031         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
6032         my $cmd = ['socat', '-T30', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=5"];
6033
6034         my $pid = fork();
6035         if (!defined($pid)) {
6036             die "forking socat tunnel failed\n";
6037         } elsif ($pid == 0) {
6038             exec(@$cmd);
6039             warn "exec failed: $!\n";
6040             POSIX::_exit(-1);
6041         }
6042         $jobs->{"drive-$drive"}->{pid} = $pid;
6043
6044         my $timeout = 0;
6045         while (!-S $unixsocket) {
6046             die "nbd connection helper timed out\n"
6047                 if $timeout++ > 5;
6048             sleep 1;
6049         }
6050     } else {
6051         my $storecfg = PVE::Storage::config();
6052         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6053
6054         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6055
6056         $format = qemu_img_format($dst_scfg, $dst_volname);
6057
6058         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6059
6060         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6061     }
6062
6063     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6064     $opts->{format} = $format if $format;
6065
6066     print "drive mirror is starting for drive-$drive\n";
6067
6068     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6069
6070     if (my $err = $@) {
6071         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6072         die "mirroring error: $err";
6073     }
6074
6075     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6076 }
6077
6078 sub qemu_drive_mirror_monitor {
6079     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6080
6081     eval {
6082         my $err_complete = 0;
6083
6084         while (1) {
6085             die "storage migration timed out\n" if $err_complete > 300;
6086
6087             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6088
6089             my $running_mirror_jobs = {};
6090             foreach my $stat (@$stats) {
6091                 next if $stat->{type} ne 'mirror';
6092                 $running_mirror_jobs->{$stat->{device}} = $stat;
6093             }
6094
6095             my $readycounter = 0;
6096
6097             foreach my $job (keys %$jobs) {
6098
6099                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6100                     print "$job : finished\n";
6101                     delete $jobs->{$job};
6102                     next;
6103                 }
6104
6105                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6106
6107                 my $busy = $running_mirror_jobs->{$job}->{busy};
6108                 my $ready = $running_mirror_jobs->{$job}->{ready};
6109                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6110                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6111                     my $remaining = $total - $transferred;
6112                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6113
6114                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6115                 }
6116
6117                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6118             }
6119
6120             last if scalar(keys %$jobs) == 0;
6121
6122             if ($readycounter == scalar(keys %$jobs)) {
6123                 print "all mirroring jobs are ready \n";
6124                 last if $skipcomplete; #do the complete later
6125
6126                 if ($vmiddst && $vmiddst != $vmid) {
6127                     my $agent_running = $qga && qga_check_running($vmid);
6128                     if ($agent_running) {
6129                         print "freeze filesystem\n";
6130                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6131                     } else {
6132                         print "suspend vm\n";
6133                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6134                     }
6135
6136                     # if we clone a disk for a new target vm, we don't switch the disk
6137                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6138
6139                     if ($agent_running) {
6140                         print "unfreeze filesystem\n";
6141                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6142                     } else {
6143                         print "resume vm\n";
6144                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6145                     }
6146
6147                     last;
6148                 } else {
6149
6150                     foreach my $job (keys %$jobs) {
6151                         # try to switch the disk if source and destination are on the same guest
6152                         print "$job: Completing block job...\n";
6153
6154                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6155                         if ($@ =~ m/cannot be completed/) {
6156                             print "$job: Block job cannot be completed, try again.\n";
6157                             $err_complete++;
6158                         }else {
6159                             print "$job: Completed successfully.\n";
6160                             $jobs->{$job}->{complete} = 1;
6161                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6162                         }
6163                     }
6164                 }
6165             }
6166             sleep 1;
6167         }
6168     };
6169     my $err = $@;
6170
6171     if ($err) {
6172         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6173         die "mirroring error: $err";
6174     }
6175
6176 }
6177
6178 sub qemu_blockjobs_cancel {
6179     my ($vmid, $jobs) = @_;
6180
6181     foreach my $job (keys %$jobs) {
6182         print "$job: Cancelling block job\n";
6183         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6184         $jobs->{$job}->{cancel} = 1;
6185     }
6186
6187     while (1) {
6188         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6189
6190         my $running_jobs = {};
6191         foreach my $stat (@$stats) {
6192             $running_jobs->{$stat->{device}} = $stat;
6193         }
6194
6195         foreach my $job (keys %$jobs) {
6196
6197             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6198                 print "$job: Done.\n";
6199                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6200                 delete $jobs->{$job};
6201             }
6202         }
6203
6204         last if scalar(keys %$jobs) == 0;
6205
6206         sleep 1;
6207     }
6208 }
6209
6210 sub qemu_blockjobs_finish_tunnel {
6211    my ($vmid, $job, $cpid) = @_;
6212
6213    return if !$cpid;
6214
6215    for (my $i = 1; $i < 20; $i++) {
6216         my $waitpid = waitpid($cpid, WNOHANG);
6217         last if (defined($waitpid) && ($waitpid == $cpid));
6218
6219         if ($i == 10) {
6220             kill(15, $cpid);
6221         } elsif ($i >= 15) {
6222             kill(9, $cpid);
6223         }
6224         sleep (1);
6225     }
6226     unlink "/run/qemu-server/$vmid.mirror-$job";
6227 }
6228
6229 sub clone_disk {
6230     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6231         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6232
6233     my $newvolid;
6234
6235     if (!$full) {
6236         print "create linked clone of drive $drivename ($drive->{file})\n";
6237         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6238         push @$newvollist, $newvolid;
6239     } else {
6240
6241         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6242         $storeid = $storage if $storage;
6243
6244         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6245         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6246
6247         print "create full clone of drive $drivename ($drive->{file})\n";
6248         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, undef, ($size/1024));
6249         push @$newvollist, $newvolid;
6250
6251         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6252
6253         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6254         if (!$running || $snapname) {
6255             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6256         } else {
6257
6258             my $kvmver = get_running_qemu_version ($vmid);
6259             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6260                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6261                     if $drive->{iothread};
6262             }
6263
6264             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6265         }
6266     }
6267
6268     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6269
6270     my $disk = $drive;
6271     $disk->{format} = undef;
6272     $disk->{file} = $newvolid;
6273     $disk->{size} = $size;
6274
6275     return $disk;
6276 }
6277
6278 # this only works if VM is running
6279 sub get_current_qemu_machine {
6280     my ($vmid) = @_;
6281
6282     my $cmd = { execute => 'query-machines', arguments => {} };
6283     my $res = vm_qmp_command($vmid, $cmd);
6284
6285     my ($current, $default);
6286     foreach my $e (@$res) {
6287         $default = $e->{name} if $e->{'is-default'};
6288         $current = $e->{name} if $e->{'is-current'};
6289     }
6290
6291     # fallback to the default machine if current is not supported by qemu
6292     return $current || $default || 'pc';
6293 }
6294
6295 sub get_running_qemu_version {
6296     my ($vmid) = @_;
6297     my $cmd = { execute => 'query-version', arguments => {} };
6298     my $res = vm_qmp_command($vmid, $cmd);
6299     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6300 }
6301
6302 sub qemu_machine_feature_enabled {
6303     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6304
6305     my $current_major;
6306     my $current_minor;
6307
6308     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6309
6310         $current_major = $3;
6311         $current_minor = $4;
6312
6313     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6314
6315         $current_major = $1;
6316         $current_minor = $2;
6317     }
6318
6319     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6320
6321
6322 }
6323
6324 sub qemu_machine_pxe {
6325     my ($vmid, $conf, $machine) = @_;
6326
6327     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6328
6329     foreach my $opt (keys %$conf) {
6330         next if $opt !~ m/^net(\d+)$/;
6331         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6332         next if !$net;
6333         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6334         return $machine.".pxe" if $romfile =~ m/pxe/;
6335         last;
6336     }
6337
6338     return $machine;
6339 }
6340
6341 sub qemu_use_old_bios_files {
6342     my ($machine_type) = @_;
6343
6344     return if !$machine_type;
6345
6346     my $use_old_bios_files = undef;
6347
6348     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6349         $machine_type = $1;
6350         $use_old_bios_files = 1;
6351     } else {
6352         my $kvmver = kvm_user_version();
6353         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6354         # load new efi bios files on migration. So this hack is required to allow
6355         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6356         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6357         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6358     }
6359
6360     return ($use_old_bios_files, $machine_type);
6361 }
6362
6363 sub create_efidisk {
6364     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6365
6366     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6367
6368     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6369     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6370     PVE::Storage::activate_volumes($storecfg, [$volid]);
6371
6372     my $path = PVE::Storage::path($storecfg, $volid);
6373     eval {
6374         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6375     };
6376     die "Copying EFI vars image failed: $@" if $@;
6377
6378     return ($volid, $vars_size);
6379 }
6380
6381 sub lspci {
6382
6383     my $devices = {};
6384
6385     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6386             my (undef, $id, $function) = @_;
6387             my $res = { id => $id, function => $function};
6388             push @{$devices->{$id}}, $res;
6389     });
6390
6391     # Entries should be sorted by functions.
6392     foreach my $id (keys %$devices) {
6393         my $dev = $devices->{$id};
6394         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6395     }
6396
6397     return $devices;
6398 }
6399
6400 sub vm_iothreads_list {
6401     my ($vmid) = @_;
6402
6403     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6404
6405     my $iothreads = {};
6406     foreach my $iothread (@$res) {
6407         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6408     }
6409
6410     return $iothreads;
6411 }
6412
6413 sub scsihw_infos {
6414     my ($conf, $drive) = @_;
6415
6416     my $maxdev = 0;
6417
6418     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6419         $maxdev = 7;
6420     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6421         $maxdev = 1;
6422     } else {
6423         $maxdev = 256;
6424     }
6425
6426     my $controller = int($drive->{index} / $maxdev);
6427     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6428
6429     return ($maxdev, $controller, $controller_prefix);
6430 }
6431
6432 sub add_hyperv_enlightenments {
6433     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6434
6435     return if $winversion < 6;
6436     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6437
6438     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6439
6440     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6441         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6442         push @$cpuFlags , 'hv_vapic';
6443         push @$cpuFlags , 'hv_time';
6444     } else {
6445         push @$cpuFlags , 'hv_spinlocks=0xffff';
6446     }
6447
6448     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6449         push @$cpuFlags , 'hv_reset';
6450         push @$cpuFlags , 'hv_vpindex';
6451         push @$cpuFlags , 'hv_runtime';
6452     }
6453
6454     if ($winversion >= 7) {
6455         push @$cpuFlags , 'hv_relaxed';
6456     }
6457 }
6458
6459 sub windows_version {
6460     my ($ostype) = @_;
6461
6462     return 0 if !$ostype;
6463
6464     my $winversion = 0;
6465
6466     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6467         $winversion = 5;
6468     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6469         $winversion = 6;
6470     } elsif ($ostype =~ m/^win(\d+)$/) {
6471         $winversion = $1;
6472     }
6473
6474     return $winversion;
6475 }
6476
6477 sub resolve_dst_disk_format {
6478         my ($storecfg, $storeid, $src_volname, $format) = @_;
6479         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6480
6481         if (!$format) {
6482             # if no target format is specified, use the source disk format as hint
6483             if ($src_volname) {
6484                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6485                 $format = qemu_img_format($scfg, $src_volname);
6486             } else {
6487                 return $defFormat;
6488             }
6489         }
6490
6491         # test if requested format is supported - else use default
6492         my $supported = grep { $_ eq $format } @$validFormats;
6493         $format = $defFormat if !$supported;
6494         return $format;
6495 }
6496
6497 sub resolve_first_disk {
6498     my $conf = shift;
6499     my @disks = PVE::QemuServer::valid_drive_names();
6500     my $firstdisk;
6501     foreach my $ds (reverse @disks) {
6502         next if !$conf->{$ds};
6503         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6504         next if PVE::QemuServer::drive_is_cdrom($disk);
6505         $firstdisk = $ds;
6506     }
6507     return $firstdisk;
6508 }
6509
6510 sub generate_smbios1_uuid {
6511     my ($uuid, $uuid_str);
6512     UUID::generate($uuid);
6513     UUID::unparse($uuid, $uuid_str);
6514     return "uuid=$uuid_str";
6515 }
6516
6517 # bash completion helper
6518
6519 sub complete_backup_archives {
6520     my ($cmdname, $pname, $cvalue) = @_;
6521
6522     my $cfg = PVE::Storage::config();
6523
6524     my $storeid;
6525
6526     if ($cvalue =~ m/^([^:]+):/) {
6527         $storeid = $1;
6528     }
6529
6530     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6531
6532     my $res = [];
6533     foreach my $id (keys %$data) {
6534         foreach my $item (@{$data->{$id}}) {
6535             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6536             push @$res, $item->{volid} if defined($item->{volid});
6537         }
6538     }
6539
6540     return $res;
6541 }
6542
6543 my $complete_vmid_full = sub {
6544     my ($running) = @_;
6545
6546     my $idlist = vmstatus();
6547
6548     my $res = [];
6549
6550     foreach my $id (keys %$idlist) {
6551         my $d = $idlist->{$id};
6552         if (defined($running)) {
6553             next if $d->{template};
6554             next if $running && $d->{status} ne 'running';
6555             next if !$running && $d->{status} eq 'running';
6556         }
6557         push @$res, $id;
6558
6559     }
6560     return $res;
6561 };
6562
6563 sub complete_vmid {
6564     return &$complete_vmid_full();
6565 }
6566
6567 sub complete_vmid_stopped {
6568     return &$complete_vmid_full(0);
6569 }
6570
6571 sub complete_vmid_running {
6572     return &$complete_vmid_full(1);
6573 }
6574
6575 sub complete_storage {
6576
6577     my $cfg = PVE::Storage::config();
6578     my $ids = $cfg->{ids};
6579
6580     my $res = [];
6581     foreach my $sid (keys %$ids) {
6582         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6583         next if !$ids->{$sid}->{content}->{images};
6584         push @$res, $sid;
6585     }
6586
6587     return $res;
6588 }
6589
6590 sub nbd_stop {
6591     my ($vmid) = @_;
6592
6593     vm_mon_cmd($vmid, 'nbd-server-stop');
6594 }
6595
6596 1;