PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42
  43 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  44
  45 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  46
  47 # Note about locking: we use flock on the config file protect
  48 # against concurent actions.
  49 # Aditionaly, we have a 'lock' setting in the config file. This
  50 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  51 # allowed when such lock is set. But you can ignore this kind of
  52 # lock with the --skiplock flag.
  53
  54 cfs_register_file('/qemu-server/',
  55                   \&parse_vm_config,
  56                   \&write_vm_config);
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  72     type => 'string',
  73     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  74     description => "The drive's backing file's data format.",
  75     optional => 1,
  76 });
  77
  78 #no warnings 'redefine';
  79
  80 sub cgroups_write {
  81    my ($controller, $vmid, $option, $value) = @_;
  82
  83    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  84    PVE::ProcFSTools::write_proc_entry($path, $value);
  85
  86 }
  87
  88 my $nodename = PVE::INotify::nodename();
  89
  90 mkdir "/etc/pve/nodes/$nodename";
  91 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  92 mkdir $confdir;
  93
  94 my $var_run_tmpdir = "/var/run/qemu-server";
  95 mkdir $var_run_tmpdir;
  96
  97 my $lock_dir = "/var/lock/qemu-server";
  98 mkdir $lock_dir;
  99
 100 my $pcisysfs = "/sys/bus/pci";
 101
 102 my $cpu_vendor_list = {
 103     # Intel CPUs
 104     486 => 'GenuineIntel',
 105     pentium => 'GenuineIntel',
 106     pentium2  => 'GenuineIntel',
 107     pentium3  => 'GenuineIntel',
 108     coreduo => 'GenuineIntel',
 109     core2duo => 'GenuineIntel',
 110     Conroe  => 'GenuineIntel',
 111     Penryn  => 'GenuineIntel',
 112     Nehalem  => 'GenuineIntel',
 113     'Nehalem-IBRS'  => 'GenuineIntel',
 114     Westmere => 'GenuineIntel',
 115     'Westmere-IBRS' => 'GenuineIntel',
 116     SandyBridge => 'GenuineIntel',
 117     'SandyBridge-IBRS' => 'GenuineIntel',
 118     IvyBridge => 'GenuineIntel',
 119     'IvyBridge-IBRS' => 'GenuineIntel',
 120     Haswell => 'GenuineIntel',
 121     'Haswell-IBRS' => 'GenuineIntel',
 122     'Haswell-noTSX' => 'GenuineIntel',
 123     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 124     Broadwell => 'GenuineIntel',
 125     'Broadwell-IBRS' => 'GenuineIntel',
 126     'Broadwell-noTSX' => 'GenuineIntel',
 127     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 128     'Skylake-Client' => 'GenuineIntel',
 129     'Skylake-Client-IBRS' => 'GenuineIntel',
 130     'Skylake-Server' => 'GenuineIntel',
 131     'Skylake-Server-IBRS' => 'GenuineIntel',
 132
 133     # AMD CPUs
 134     athlon => 'AuthenticAMD',
 135     phenom  => 'AuthenticAMD',
 136     Opteron_G1  => 'AuthenticAMD',
 137     Opteron_G2  => 'AuthenticAMD',
 138     Opteron_G3  => 'AuthenticAMD',
 139     Opteron_G4  => 'AuthenticAMD',
 140     Opteron_G5  => 'AuthenticAMD',
 141
 142     # generic types, use vendor from host node
 143     host => 'default',
 144     kvm32 => 'default',
 145     kvm64 => 'default',
 146     qemu32 => 'default',
 147     qemu64 => 'default',
 148     max => 'default',
 149 };
 150
 151 my $cpu_flag = qr/[+-](pcid|spec-ctrl)/;
 152
 153 my $cpu_fmt = {
 154     cputype => {
 155         description => "Emulated CPU type.",
 156         type => 'string',
 157         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 158         default => 'kvm64',
 159         default_key => 1,
 160     },
 161     hidden => {
 162         description => "Do not identify as a KVM virtual machine.",
 163         type => 'boolean',
 164         optional => 1,
 165         default => 0
 166     },
 167     flags => {
 168         description => "List of additional CPU flags separated by ';'."
 169                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 170                      . " Currently supported flags: 'pcid', 'spec-ctrl'.",
 171         format_description => '+FLAG[;-FLAG...]',
 172         type => 'string',
 173         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 174         optional => 1,
 175     },
 176 };
 177
 178 my $watchdog_fmt = {
 179     model => {
 180         default_key => 1,
 181         type => 'string',
 182         enum => [qw(i6300esb ib700)],
 183         description => "Watchdog type to emulate.",
 184         default => 'i6300esb',
 185         optional => 1,
 186     },
 187     action => {
 188         type => 'string',
 189         enum => [qw(reset shutdown poweroff pause debug none)],
 190         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 191         optional => 1,
 192     },
 193 };
 194 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 195
 196 my $confdesc = {
 197     onboot => {
 198         optional => 1,
 199         type => 'boolean',
 200         description => "Specifies whether a VM will be started during system bootup.",
 201         default => 0,
 202     },
 203     autostart => {
 204         optional => 1,
 205         type => 'boolean',
 206         description => "Automatic restart after crash (currently ignored).",
 207         default => 0,
 208     },
 209     hotplug => {
 210         optional => 1,
 211         type => 'string', format => 'pve-hotplug-features',
 212         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 213         default => 'network,disk,usb',
 214     },
 215     reboot => {
 216         optional => 1,
 217         type => 'boolean',
 218         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 219         default => 1,
 220     },
 221     lock => {
 222         optional => 1,
 223         type => 'string',
 224         description => "Lock/unlock the VM.",
 225         enum => [qw(migrate backup snapshot rollback)],
 226     },
 227     cpulimit => {
 228         optional => 1,
 229         type => 'number',
 230         description => "Limit of CPU usage.",
 231         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 232         minimum => 0,
 233         maximum => 128,
 234         default => 0,
 235     },
 236     cpuunits => {
 237         optional => 1,
 238         type => 'integer',
 239         description => "CPU weight for a VM.",
 240         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 241         minimum => 2,
 242         maximum => 262144,
 243         default => 1024,
 244     },
 245     memory => {
 246         optional => 1,
 247         type => 'integer',
 248         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 249         minimum => 16,
 250         default => 512,
 251     },
 252     balloon => {
 253         optional => 1,
 254         type => 'integer',
 255         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 256         minimum => 0,
 257     },
 258     shares => {
 259         optional => 1,
 260         type => 'integer',
 261         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 262         minimum => 0,
 263         maximum => 50000,
 264         default => 1000,
 265     },
 266     keyboard => {
 267         optional => 1,
 268         type => 'string',
 269         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.".
 270                        "It should not be necessary to set it.",
 271         enum => PVE::Tools::kvmkeymaplist(),
 272         default => undef,
 273     },
 274     name => {
 275         optional => 1,
 276         type => 'string', format => 'dns-name',
 277         description => "Set a name for the VM. Only used on the configuration web interface.",
 278     },
 279     scsihw => {
 280         optional => 1,
 281         type => 'string',
 282         description => "SCSI controller model",
 283         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 284         default => 'lsi',
 285     },
 286     description => {
 287         optional => 1,
 288         type => 'string',
 289         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 290     },
 291     ostype => {
 292         optional => 1,
 293         type => 'string',
 294         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 295         description => "Specify guest operating system.",
 296         verbose_description => <<EODESC,
 297 Specify guest operating system. This is used to enable special
 298 optimization/features for specific operating systems:
 299
 300 [horizontal]
 301 other;; unspecified OS
 302 wxp;; Microsoft Windows XP
 303 w2k;; Microsoft Windows 2000
 304 w2k3;; Microsoft Windows 2003
 305 w2k8;; Microsoft Windows 2008
 306 wvista;; Microsoft Windows Vista
 307 win7;; Microsoft Windows 7
 308 win8;; Microsoft Windows 8/2012/2012r2
 309 win10;; Microsoft Windows 10/2016
 310 l24;; Linux 2.4 Kernel
 311 l26;; Linux 2.6/3.X Kernel
 312 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 313 EODESC
 314     },
 315     boot => {
 316         optional => 1,
 317         type => 'string',
 318         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 319         pattern => '[acdn]{1,4}',
 320         default => 'cdn',
 321     },
 322     bootdisk => {
 323         optional => 1,
 324         type => 'string', format => 'pve-qm-bootdisk',
 325         description => "Enable booting from specified disk.",
 326         pattern => '(ide|sata|scsi|virtio)\d+',
 327     },
 328     smp => {
 329         optional => 1,
 330         type => 'integer',
 331         description => "The number of CPUs. Please use option -sockets instead.",
 332         minimum => 1,
 333         default => 1,
 334     },
 335     sockets => {
 336         optional => 1,
 337         type => 'integer',
 338         description => "The number of CPU sockets.",
 339         minimum => 1,
 340         default => 1,
 341     },
 342     cores => {
 343         optional => 1,
 344         type => 'integer',
 345         description => "The number of cores per socket.",
 346         minimum => 1,
 347         default => 1,
 348     },
 349     numa => {
 350         optional => 1,
 351         type => 'boolean',
 352         description => "Enable/disable NUMA.",
 353         default => 0,
 354     },
 355     hugepages => {
 356         optional => 1,
 357         type => 'string',
 358         description => "Enable/disable hugepages memory.",
 359         enum => [qw(any 2 1024)],
 360     },
 361     vcpus => {
 362         optional => 1,
 363         type => 'integer',
 364         description => "Number of hotplugged vcpus.",
 365         minimum => 1,
 366         default => 0,
 367     },
 368     acpi => {
 369         optional => 1,
 370         type => 'boolean',
 371         description => "Enable/disable ACPI.",
 372         default => 1,
 373     },
 374     agent => {
 375         optional => 1,
 376         type => 'boolean',
 377         description => "Enable/disable Qemu GuestAgent.",
 378         default => 0,
 379     },
 380     kvm => {
 381         optional => 1,
 382         type => 'boolean',
 383         description => "Enable/disable KVM hardware virtualization.",
 384         default => 1,
 385     },
 386     tdf => {
 387         optional => 1,
 388         type => 'boolean',
 389         description => "Enable/disable time drift fix.",
 390         default => 0,
 391     },
 392     localtime => {
 393         optional => 1,
 394         type => 'boolean',
 395         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 396     },
 397     freeze => {
 398         optional => 1,
 399         type => 'boolean',
 400         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 401     },
 402     vga => {
 403         optional => 1,
 404         type => 'string',
 405         description => "Select the VGA type.",
 406         verbose_description => "Select the VGA type. If you want to use high resolution" .
 407             " modes (>= 1280x1024x16) then you should use the options " .
 408             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 409             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 410             "display sever. For win* OS you can select how many independent " .
 411             "displays you want, Linux guests can add displays them self. " .
 412             "You can also run without any graphic card, using a serial device" .
 413             " as terminal.",
 414         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 415     },
 416     watchdog => {
 417         optional => 1,
 418         type => 'string', format => 'pve-qm-watchdog',
 419         description => "Create a virtual hardware watchdog device.",
 420         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 421             " (by a guest action), the watchdog must be periodically polled " .
 422             "by an agent inside the guest or else the watchdog will reset " .
 423             "the guest (or execute the respective action specified)",
 424     },
 425     startdate => {
 426         optional => 1,
 427         type => 'string',
 428         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 429         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 430         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 431         default => 'now',
 432     },
 433     startup =>  get_standard_option('pve-startup-order'),
 434     template => {
 435         optional => 1,
 436         type => 'boolean',
 437         description => "Enable/disable Template.",
 438         default => 0,
 439     },
 440     args => {
 441         optional => 1,
 442         type => 'string',
 443         description => "Arbitrary arguments passed to kvm.",
 444         verbose_description => <<EODESCR,
 445 Arbitrary arguments passed to kvm, for example:
 446
 447 args: -no-reboot -no-hpet
 448
 449 NOTE: this option is for experts only.
 450 EODESCR
 451     },
 452     tablet => {
 453         optional => 1,
 454         type => 'boolean',
 455         default => 1,
 456         description => "Enable/disable the USB tablet device.",
 457         verbose_description => "Enable/disable the USB tablet device. This device is " .
 458             "usually needed to allow absolute mouse positioning with VNC. " .
 459             "Else the mouse runs out of sync with normal VNC clients. " .
 460             "If you're running lots of console-only guests on one host, " .
 461             "you may consider disabling this to save some context switches. " .
 462             "This is turned off by default if you use spice (-vga=qxl).",
 463     },
 464     migrate_speed => {
 465         optional => 1,
 466         type => 'integer',
 467         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 468         minimum => 0,
 469         default => 0,
 470     },
 471     migrate_downtime => {
 472         optional => 1,
 473         type => 'number',
 474         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 475         minimum => 0,
 476         default => 0.1,
 477     },
 478     cdrom => {
 479         optional => 1,
 480         type => 'string', format => 'pve-qm-ide',
 481         typetext => '<volume>',
 482         description => "This is an alias for option -ide2",
 483     },
 484     cpu => {
 485         optional => 1,
 486         description => "Emulated CPU type.",
 487         type => 'string',
 488         format => $cpu_fmt,
 489     },
 490     parent => get_standard_option('pve-snapshot-name', {
 491         optional => 1,
 492         description => "Parent snapshot name. This is used internally, and should not be modified.",
 493     }),
 494     snaptime => {
 495         optional => 1,
 496         description => "Timestamp for snapshots.",
 497         type => 'integer',
 498         minimum => 0,
 499     },
 500     vmstate => {
 501         optional => 1,
 502         type => 'string', format => 'pve-volume-id',
 503         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 504     },
 505     vmstatestorage => get_standard_option('pve-storage-id', {
 506         description => "Default storage for VM state volumes/files.",
 507         optional => 1,
 508     }),
 509     machine => {
 510         description => "Specific the Qemu machine type.",
 511         type => 'string',
 512         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 513         maxLength => 40,
 514         optional => 1,
 515     },
 516     smbios1 => {
 517         description => "Specify SMBIOS type 1 fields.",
 518         type => 'string', format => 'pve-qm-smbios1',
 519         maxLength => 256,
 520         optional => 1,
 521     },
 522     protection => {
 523         optional => 1,
 524         type => 'boolean',
 525         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 526         default => 0,
 527     },
 528     bios => {
 529         optional => 1,
 530         type => 'string',
 531         enum => [ qw(seabios ovmf) ],
 532         description => "Select BIOS implementation.",
 533         default => 'seabios',
 534     },
 535 };
 536
 537 # what about other qemu settings ?
 538 #cpu => 'string',
 539 #machine => 'string',
 540 #fda => 'file',
 541 #fdb => 'file',
 542 #mtdblock => 'file',
 543 #sd => 'file',
 544 #pflash => 'file',
 545 #snapshot => 'bool',
 546 #bootp => 'file',
 547 ##tftp => 'dir',
 548 ##smb => 'dir',
 549 #kernel => 'file',
 550 #append => 'string',
 551 #initrd => 'file',
 552 ##soundhw => 'string',
 553
 554 while (my ($k, $v) = each %$confdesc) {
 555     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 556 }
 557
 558 my $MAX_IDE_DISKS = 4;
 559 my $MAX_SCSI_DISKS = 14;
 560 my $MAX_VIRTIO_DISKS = 16;
 561 my $MAX_SATA_DISKS = 6;
 562 my $MAX_USB_DEVICES = 5;
 563 my $MAX_NETS = 32;
 564 my $MAX_UNUSED_DISKS = 8;
 565 my $MAX_HOSTPCI_DEVICES = 4;
 566 my $MAX_SERIAL_PORTS = 4;
 567 my $MAX_PARALLEL_PORTS = 3;
 568 my $MAX_NUMA = 8;
 569
 570 my $numa_fmt = {
 571     cpus => {
 572         type => "string",
 573         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 574         description => "CPUs accessing this NUMA node.",
 575         format_description => "id[-id];...",
 576     },
 577     memory => {
 578         type => "number",
 579         description => "Amount of memory this NUMA node provides.",
 580         optional => 1,
 581     },
 582     hostnodes => {
 583         type => "string",
 584         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 585         description => "Host NUMA nodes to use.",
 586         format_description => "id[-id];...",
 587         optional => 1,
 588     },
 589     policy => {
 590         type => 'string',
 591         enum => [qw(preferred bind interleave)],
 592         description => "NUMA allocation policy.",
 593         optional => 1,
 594     },
 595 };
 596 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 597 my $numadesc = {
 598     optional => 1,
 599     type => 'string', format => $numa_fmt,
 600     description => "NUMA topology.",
 601 };
 602 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 603
 604 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 605     $confdesc->{"numa$i"} = $numadesc;
 606 }
 607
 608 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 609                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 610                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 611 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 612
 613 my $net_fmt_bridge_descr = <<__EOD__;
 614 Bridge to attach the network device to. The Proxmox VE standard bridge
 615 is called 'vmbr0'.
 616
 617 If you do not specify a bridge, we create a kvm user (NATed) network
 618 device, which provides DHCP and DNS services. The following addresses
 619 are used:
 620
 621  10.0.2.2   Gateway
 622  10.0.2.3   DNS Server
 623  10.0.2.4   SMB Server
 624
 625 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 626 __EOD__
 627
 628 my $net_fmt = {
 629     macaddr => {
 630         type => 'string',
 631         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 632         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 633         format_description => "XX:XX:XX:XX:XX:XX",
 634         optional => 1,
 635     },
 636     model => {
 637         type => 'string',
 638         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 639         enum => $nic_model_list,
 640         default_key => 1,
 641     },
 642     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 643     bridge => {
 644         type => 'string',
 645         description => $net_fmt_bridge_descr,
 646         format_description => 'bridge',
 647         optional => 1,
 648     },
 649     queues => {
 650         type => 'integer',
 651         minimum => 0, maximum => 16,
 652         description => 'Number of packet queues to be used on the device.',
 653         optional => 1,
 654     },
 655     rate => {
 656         type => 'number',
 657         minimum => 0,
 658         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 659         optional => 1,
 660     },
 661     tag => {
 662         type => 'integer',
 663         minimum => 1, maximum => 4094,
 664         description => 'VLAN tag to apply to packets on this interface.',
 665         optional => 1,
 666     },
 667     trunks => {
 668         type => 'string',
 669         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 670         description => 'VLAN trunks to pass through this interface.',
 671         format_description => 'vlanid[;vlanid...]',
 672         optional => 1,
 673     },
 674     firewall => {
 675         type => 'boolean',
 676         description => 'Whether this interface should be protected by the firewall.',
 677         optional => 1,
 678     },
 679     link_down => {
 680         type => 'boolean',
 681         description => 'Whether this interface should be disconnected (like pulling the plug).',
 682         optional => 1,
 683     },
 684 };
 685
 686 my $netdesc = {
 687     optional => 1,
 688     type => 'string', format => $net_fmt,
 689     description => "Specify network devices.",
 690 };
 691
 692 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 693
 694 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 695     $confdesc->{"net$i"} = $netdesc;
 696 }
 697
 698 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 699 sub verify_volume_id_or_qm_path {
 700     my ($volid, $noerr) = @_;
 701
 702     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 703         return $volid;
 704     }
 705
 706     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 707     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 708     if ($@) {
 709         return undef if $noerr;
 710         die $@;
 711     }
 712     return $volid;
 713 }
 714
 715 my $drivename_hash;
 716
 717 my %drivedesc_base = (
 718     volume => { alias => 'file' },
 719     file => {
 720         type => 'string',
 721         format => 'pve-volume-id-or-qm-path',
 722         default_key => 1,
 723         format_description => 'volume',
 724         description => "The drive's backing volume.",
 725     },
 726     media => {
 727         type => 'string',
 728         enum => [qw(cdrom disk)],
 729         description => "The drive's media type.",
 730         default => 'disk',
 731         optional => 1
 732     },
 733     cyls => {
 734         type => 'integer',
 735         description => "Force the drive's physical geometry to have a specific cylinder count.",
 736         optional => 1
 737     },
 738     heads => {
 739         type => 'integer',
 740         description => "Force the drive's physical geometry to have a specific head count.",
 741         optional => 1
 742     },
 743     secs => {
 744         type => 'integer',
 745         description => "Force the drive's physical geometry to have a specific sector count.",
 746         optional => 1
 747     },
 748     trans => {
 749         type => 'string',
 750         enum => [qw(none lba auto)],
 751         description => "Force disk geometry bios translation mode.",
 752         optional => 1,
 753     },
 754     snapshot => {
 755         type => 'boolean',
 756         description => "Whether the drive should be included when making snapshots.",
 757         optional => 1,
 758     },
 759     cache => {
 760         type => 'string',
 761         enum => [qw(none writethrough writeback unsafe directsync)],
 762         description => "The drive's cache mode",
 763         optional => 1,
 764     },
 765     format => get_standard_option('pve-qm-image-format'),
 766     size => {
 767         type => 'string',
 768         format => 'disk-size',
 769         format_description => 'DiskSize',
 770         description => "Disk size. This is purely informational and has no effect.",
 771         optional => 1,
 772     },
 773     backup => {
 774         type => 'boolean',
 775         description => "Whether the drive should be included when making backups.",
 776         optional => 1,
 777     },
 778     replicate => {
 779         type => 'boolean',
 780         description => 'Whether the drive should considered for replication jobs.',
 781         optional => 1,
 782         default => 1,
 783     },
 784     rerror => {
 785         type => 'string',
 786         enum => [qw(ignore report stop)],
 787         description => 'Read error action.',
 788         optional => 1,
 789     },
 790     werror => {
 791         type => 'string',
 792         enum => [qw(enospc ignore report stop)],
 793         description => 'Write error action.',
 794         optional => 1,
 795     },
 796     aio => {
 797         type => 'string',
 798         enum => [qw(native threads)],
 799         description => 'AIO type to use.',
 800         optional => 1,
 801     },
 802     discard => {
 803         type => 'string',
 804         enum => [qw(ignore on)],
 805         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 806         optional => 1,
 807     },
 808     detect_zeroes => {
 809         type => 'boolean',
 810         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 811         optional => 1,
 812     },
 813     serial => {
 814         type => 'string',
 815         format => 'urlencoded',
 816         format_description => 'serial',
 817         maxLength => 20*3, # *3 since it's %xx url enoded
 818         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 819         optional => 1,
 820     }
 821 );
 822
 823 my %iothread_fmt = ( iothread => {
 824         type => 'boolean',
 825         description => "Whether to use iothreads for this drive",
 826         optional => 1,
 827 });
 828
 829 my %model_fmt = (
 830     model => {
 831         type => 'string',
 832         format => 'urlencoded',
 833         format_description => 'model',
 834         maxLength => 40*3, # *3 since it's %xx url enoded
 835         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 836         optional => 1,
 837     },
 838 );
 839
 840 my %queues_fmt = (
 841     queues => {
 842         type => 'integer',
 843         description => "Number of queues.",
 844         minimum => 2,
 845         optional => 1
 846     }
 847 );
 848
 849 my %scsiblock_fmt = (
 850     scsiblock => {
 851         type => 'boolean',
 852         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 853         optional => 1,
 854         default => 0,
 855     },
 856 );
 857
 858 my $add_throttle_desc = sub {
 859     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 860     my $d = {
 861         type => $type,
 862         format_description => $unit,
 863         description => "Maximum $what in $longunit.",
 864         optional => 1,
 865     };
 866     $d->{minimum} = $minimum if defined($minimum);
 867     $drivedesc_base{$key} = $d;
 868 };
 869 # throughput: (leaky bucket)
 870 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 871 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 872 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 873 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 874 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 875 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 876 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 877 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 878 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 879
 880 # pools: (pool of IO before throttling starts taking effect)
 881 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 882 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 883 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 884 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 885 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 886 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 887
 888 # burst lengths
 889 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 890 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 891 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 892 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 893 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 894 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 895
 896 # legacy support
 897 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
 898 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
 899 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
 900 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
 901
 902 my $ide_fmt = {
 903     %drivedesc_base,
 904     %model_fmt,
 905 };
 906 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 907
 908 my $idedesc = {
 909     optional => 1,
 910     type => 'string', format => $ide_fmt,
 911     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 912 };
 913 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 914
 915 my $scsi_fmt = {
 916     %drivedesc_base,
 917     %iothread_fmt,
 918     %queues_fmt,
 919     %scsiblock_fmt,
 920 };
 921 my $scsidesc = {
 922     optional => 1,
 923     type => 'string', format => $scsi_fmt,
 924     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 925 };
 926 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 927
 928 my $sata_fmt = {
 929     %drivedesc_base,
 930 };
 931 my $satadesc = {
 932     optional => 1,
 933     type => 'string', format => $sata_fmt,
 934     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 935 };
 936 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 937
 938 my $virtio_fmt = {
 939     %drivedesc_base,
 940     %iothread_fmt,
 941 };
 942 my $virtiodesc = {
 943     optional => 1,
 944     type => 'string', format => $virtio_fmt,
 945     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 946 };
 947 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 948
 949 my $alldrive_fmt = {
 950     %drivedesc_base,
 951     %iothread_fmt,
 952     %model_fmt,
 953     %queues_fmt,
 954     %scsiblock_fmt,
 955 };
 956
 957 my $efidisk_fmt = {
 958     volume => { alias => 'file' },
 959     file => {
 960         type => 'string',
 961         format => 'pve-volume-id-or-qm-path',
 962         default_key => 1,
 963         format_description => 'volume',
 964         description => "The drive's backing volume.",
 965     },
 966     format => get_standard_option('pve-qm-image-format'),
 967     size => {
 968         type => 'string',
 969         format => 'disk-size',
 970         format_description => 'DiskSize',
 971         description => "Disk size. This is purely informational and has no effect.",
 972         optional => 1,
 973     },
 974 };
 975
 976 my $efidisk_desc = {
 977     optional => 1,
 978     type => 'string', format => $efidisk_fmt,
 979     description => "Configure a Disk for storing EFI vars",
 980 };
 981
 982 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 983
 984 my $usb_fmt = {
 985     host => {
 986         default_key => 1,
 987         type => 'string', format => 'pve-qm-usb-device',
 988         format_description => 'HOSTUSBDEVICE|spice',
 989         description => <<EODESCR,
 990 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 991
 992  'bus-port(.port)*' (decimal numbers) or
 993  'vendor_id:product_id' (hexadeciaml numbers) or
 994  'spice'
 995
 996 You can use the 'lsusb -t' command to list existing usb devices.
 997
 998 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 999
1000 The value 'spice' can be used to add a usb redirection devices for spice.
1001 EODESCR
1002     },
1003     usb3 => {
1004         optional => 1,
1005         type => 'boolean',
1006         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1007         default => 0,
1008     },
1009 };
1010
1011 my $usbdesc = {
1012     optional => 1,
1013     type => 'string', format => $usb_fmt,
1014     description => "Configure an USB device (n is 0 to 4).",
1015 };
1016 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1017
1018 # NOTE: the match-groups of this regex are used in parse_hostpci
1019 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1020 my $hostpci_fmt = {
1021     host => {
1022         default_key => 1,
1023         type => 'string',
1024         pattern => qr/$PCIRE(;$PCIRE)*/,
1025         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1026         description => <<EODESCR,
1027 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1028 of PCI virtual functions of the host. HOSTPCIID syntax is:
1029
1030 'bus:dev.func' (hexadecimal numbers)
1031
1032 You can us the 'lspci' command to list existing PCI devices.
1033 EODESCR
1034     },
1035     rombar => {
1036         type => 'boolean',
1037         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1038         optional => 1,
1039         default => 1,
1040     },
1041     romfile => {
1042         type => 'string',
1043         pattern => '[^,;]+',
1044         format_description => 'string',
1045         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1046         optional => 1,
1047     },
1048     pcie => {
1049         type => 'boolean',
1050         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1051         optional => 1,
1052         default => 0,
1053     },
1054     'x-vga' => {
1055         type => 'boolean',
1056         description =>  "Enable vfio-vga device support.",
1057         optional => 1,
1058         default => 0,
1059     },
1060 };
1061 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1062
1063 my $hostpcidesc = {
1064         optional => 1,
1065         type => 'string', format => 'pve-qm-hostpci',
1066         description => "Map host PCI devices into guest.",
1067         verbose_description =>  <<EODESCR,
1068 Map host PCI devices into guest.
1069
1070 NOTE: This option allows direct access to host hardware. So it is no longer
1071 possible to migrate such machines - use with special care.
1072
1073 CAUTION: Experimental! User reported problems with this option.
1074 EODESCR
1075 };
1076 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1077
1078 my $serialdesc = {
1079         optional => 1,
1080         type => 'string',
1081         pattern => '(/dev/.+|socket)',
1082         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1083         verbose_description =>  <<EODESCR,
1084 Create a serial device inside the VM (n is 0 to 3), and pass through a
1085 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1086 host side (use 'qm terminal' to open a terminal connection).
1087
1088 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1089
1090 CAUTION: Experimental! User reported problems with this option.
1091 EODESCR
1092 };
1093
1094 my $paralleldesc= {
1095         optional => 1,
1096         type => 'string',
1097         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1098         description =>  "Map host parallel devices (n is 0 to 2).",
1099         verbose_description =>  <<EODESCR,
1100 Map host parallel devices (n is 0 to 2).
1101
1102 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1103
1104 CAUTION: Experimental! User reported problems with this option.
1105 EODESCR
1106 };
1107
1108 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1109     $confdesc->{"parallel$i"} = $paralleldesc;
1110 }
1111
1112 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1113     $confdesc->{"serial$i"} = $serialdesc;
1114 }
1115
1116 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1117     $confdesc->{"hostpci$i"} = $hostpcidesc;
1118 }
1119
1120 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1121     $drivename_hash->{"ide$i"} = 1;
1122     $confdesc->{"ide$i"} = $idedesc;
1123 }
1124
1125 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1126     $drivename_hash->{"sata$i"} = 1;
1127     $confdesc->{"sata$i"} = $satadesc;
1128 }
1129
1130 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1131     $drivename_hash->{"scsi$i"} = 1;
1132     $confdesc->{"scsi$i"} = $scsidesc ;
1133 }
1134
1135 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1136     $drivename_hash->{"virtio$i"} = 1;
1137     $confdesc->{"virtio$i"} = $virtiodesc;
1138 }
1139
1140 $drivename_hash->{efidisk0} = 1;
1141 $confdesc->{efidisk0} = $efidisk_desc;
1142
1143 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1144     $confdesc->{"usb$i"} = $usbdesc;
1145 }
1146
1147 my $unuseddesc = {
1148     optional => 1,
1149     type => 'string', format => 'pve-volume-id',
1150     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1151 };
1152
1153 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1154     $confdesc->{"unused$i"} = $unuseddesc;
1155 }
1156
1157 my $kvm_api_version = 0;
1158
1159 sub kvm_version {
1160
1161     return $kvm_api_version if $kvm_api_version;
1162
1163     my $fh = IO::File->new("</dev/kvm") ||
1164         return 0;
1165
1166     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1167         $kvm_api_version = $v;
1168     }
1169
1170     $fh->close();
1171
1172     return  $kvm_api_version;
1173 }
1174
1175 my $kvm_user_version;
1176
1177 sub kvm_user_version {
1178
1179     return $kvm_user_version if $kvm_user_version;
1180
1181     $kvm_user_version = 'unknown';
1182
1183     my $code = sub {
1184         my $line = shift;
1185         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1186             $kvm_user_version = $2;
1187         }
1188     };
1189
1190     eval { run_command("kvm -version", outfunc => $code); };
1191     warn $@ if $@;
1192
1193     return $kvm_user_version;
1194
1195 }
1196
1197 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1198
1199 sub valid_drive_names {
1200     # order is important - used to autoselect boot disk
1201     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1202             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1203             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1204             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1205             'efidisk0');
1206 }
1207
1208 sub is_valid_drivename {
1209     my $dev = shift;
1210
1211     return defined($drivename_hash->{$dev});
1212 }
1213
1214 sub option_exists {
1215     my $key = shift;
1216     return defined($confdesc->{$key});
1217 }
1218
1219 sub nic_models {
1220     return $nic_model_list;
1221 }
1222
1223 sub os_list_description {
1224
1225     return {
1226         other => 'Other',
1227         wxp => 'Windows XP',
1228         w2k => 'Windows 2000',
1229         w2k3 =>, 'Windows 2003',
1230         w2k8 => 'Windows 2008',
1231         wvista => 'Windows Vista',
1232         win7 => 'Windows 7',
1233         win8 => 'Windows 8/2012',
1234         win10 => 'Windows 10/2016',
1235         l24 => 'Linux 2.4',
1236         l26 => 'Linux 2.6',
1237     };
1238 }
1239
1240 my $cdrom_path;
1241
1242 sub get_cdrom_path {
1243
1244     return  $cdrom_path if $cdrom_path;
1245
1246     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1247     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1248     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1249 }
1250
1251 sub get_iso_path {
1252     my ($storecfg, $vmid, $cdrom) = @_;
1253
1254     if ($cdrom eq 'cdrom') {
1255         return get_cdrom_path();
1256     } elsif ($cdrom eq 'none') {
1257         return '';
1258     } elsif ($cdrom =~ m|^/|) {
1259         return $cdrom;
1260     } else {
1261         return PVE::Storage::path($storecfg, $cdrom);
1262     }
1263 }
1264
1265 # try to convert old style file names to volume IDs
1266 sub filename_to_volume_id {
1267     my ($vmid, $file, $media) = @_;
1268
1269     if (!($file eq 'none' || $file eq 'cdrom' ||
1270           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1271
1272         return undef if $file =~ m|/|;
1273
1274         if ($media && $media eq 'cdrom') {
1275             $file = "local:iso/$file";
1276         } else {
1277             $file = "local:$vmid/$file";
1278         }
1279     }
1280
1281     return $file;
1282 }
1283
1284 sub verify_media_type {
1285     my ($opt, $vtype, $media) = @_;
1286
1287     return if !$media;
1288
1289     my $etype;
1290     if ($media eq 'disk') {
1291         $etype = 'images';
1292     } elsif ($media eq 'cdrom') {
1293         $etype = 'iso';
1294     } else {
1295         die "internal error";
1296     }
1297
1298     return if ($vtype eq $etype);
1299
1300     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1301 }
1302
1303 sub cleanup_drive_path {
1304     my ($opt, $storecfg, $drive) = @_;
1305
1306     # try to convert filesystem paths to volume IDs
1307
1308     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1309         ($drive->{file} !~ m|^/dev/.+|) &&
1310         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1311         ($drive->{file} !~ m/^\d+$/)) {
1312         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1313         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1314         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1315         verify_media_type($opt, $vtype, $drive->{media});
1316         $drive->{file} = $volid;
1317     }
1318
1319     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1320 }
1321
1322 sub parse_hotplug_features {
1323     my ($data) = @_;
1324
1325     my $res = {};
1326
1327     return $res if $data eq '0';
1328
1329     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1330
1331     foreach my $feature (PVE::Tools::split_list($data)) {
1332         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1333             $res->{$1} = 1;
1334         } else {
1335             die "invalid hotplug feature '$feature'\n";
1336         }
1337     }
1338     return $res;
1339 }
1340
1341 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1342 sub pve_verify_hotplug_features {
1343     my ($value, $noerr) = @_;
1344
1345     return $value if parse_hotplug_features($value);
1346
1347     return undef if $noerr;
1348
1349     die "unable to parse hotplug option\n";
1350 }
1351
1352 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1353 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1354 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1355 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1356 #        [,iothread=on][,serial=serial][,model=model]
1357
1358 sub parse_drive {
1359     my ($key, $data) = @_;
1360
1361     my ($interface, $index);
1362
1363     if ($key =~ m/^([^\d]+)(\d+)$/) {
1364         $interface = $1;
1365         $index = $2;
1366     } else {
1367         return undef;
1368     }
1369
1370     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1371                                      : $confdesc->{$key}->{format};
1372     if (!$desc) {
1373         warn "invalid drive key: $key\n";
1374         return undef;
1375     }
1376     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1377     return undef if !$res;
1378     $res->{interface} = $interface;
1379     $res->{index} = $index;
1380
1381     my $error = 0;
1382     foreach my $opt (qw(bps bps_rd bps_wr)) {
1383         if (my $bps = defined(delete $res->{$opt})) {
1384             if (defined($res->{"m$opt"})) {
1385                 warn "both $opt and m$opt specified\n";
1386                 ++$error;
1387                 next;
1388             }
1389             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1390         }
1391     }
1392
1393     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1394     for my $requirement (
1395         [mbps_max => 'mbps'],
1396         [mbps_rd_max => 'mbps_rd'],
1397         [mbps_wr_max => 'mbps_wr'],
1398         [miops_max => 'miops'],
1399         [miops_rd_max => 'miops_rd'],
1400         [miops_wr_max => 'miops_wr'],
1401         [bps_max_length => 'mbps_max'],
1402         [bps_rd_max_length => 'mbps_rd_max'],
1403         [bps_wr_max_length => 'mbps_wr_max'],
1404         [iops_max_length => 'iops_max'],
1405         [iops_rd_max_length => 'iops_rd_max'],
1406         [iops_wr_max_length => 'iops_wr_max']) {
1407         my ($option, $requires) = @$requirement;
1408         if ($res->{$option} && !$res->{$requires}) {
1409             warn "$option requires $requires\n";
1410             ++$error;
1411         }
1412     }
1413
1414     return undef if $error;
1415
1416     return undef if $res->{mbps_rd} && $res->{mbps};
1417     return undef if $res->{mbps_wr} && $res->{mbps};
1418     return undef if $res->{iops_rd} && $res->{iops};
1419     return undef if $res->{iops_wr} && $res->{iops};
1420
1421     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1422         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1423         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1424         return undef if $res->{interface} eq 'virtio';
1425     }
1426
1427     if (my $size = $res->{size}) {
1428         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1429     }
1430
1431     return $res;
1432 }
1433
1434 sub print_drive {
1435     my ($vmid, $drive) = @_;
1436     my $data = { %$drive };
1437     delete $data->{$_} for qw(index interface);
1438     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1439 }
1440
1441 sub scsi_inquiry {
1442     my($fh, $noerr) = @_;
1443
1444     my $SG_IO = 0x2285;
1445     my $SG_GET_VERSION_NUM = 0x2282;
1446
1447     my $versionbuf = "\x00" x 8;
1448     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1449     if (!$ret) {
1450         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1451         return undef;
1452     }
1453     my $version = unpack("I", $versionbuf);
1454     if ($version < 30000) {
1455         die "scsi generic interface too old\n"  if !$noerr;
1456         return undef;
1457     }
1458
1459     my $buf = "\x00" x 36;
1460     my $sensebuf = "\x00" x 8;
1461     my $cmd = pack("C x3 C x1", 0x12, 36);
1462
1463     # see /usr/include/scsi/sg.h
1464     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1465
1466     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1467                       length($sensebuf), 0, length($buf), $buf,
1468                       $cmd, $sensebuf, 6000);
1469
1470     $ret = ioctl($fh, $SG_IO, $packet);
1471     if (!$ret) {
1472         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1473         return undef;
1474     }
1475
1476     my @res = unpack($sg_io_hdr_t, $packet);
1477     if ($res[17] || $res[18]) {
1478         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1479         return undef;
1480     }
1481
1482     my $res = {};
1483     (my $byte0, my $byte1, $res->{vendor},
1484      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1485
1486     $res->{removable} = $byte1 & 128 ? 1 : 0;
1487     $res->{type} = $byte0 & 31;
1488
1489     return $res;
1490 }
1491
1492 sub path_is_scsi {
1493     my ($path) = @_;
1494
1495     my $fh = IO::File->new("+<$path") || return undef;
1496     my $res = scsi_inquiry($fh, 1);
1497     close($fh);
1498
1499     return $res;
1500 }
1501
1502 sub machine_type_is_q35 {
1503     my ($conf) = @_;
1504
1505     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1506 }
1507
1508 sub print_tabletdevice_full {
1509     my ($conf) = @_;
1510
1511     my $q35 = machine_type_is_q35($conf);
1512
1513     # we use uhci for old VMs because tablet driver was buggy in older qemu
1514     my $usbbus = $q35 ? "ehci" : "uhci";
1515
1516     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1517 }
1518
1519 sub print_drivedevice_full {
1520     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1521
1522     my $device = '';
1523     my $maxdev = 0;
1524
1525     if ($drive->{interface} eq 'virtio') {
1526         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1527         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1528         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1529     } elsif ($drive->{interface} eq 'scsi') {
1530
1531         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1532         my $unit = $drive->{index} % $maxdev;
1533         my $devicetype = 'hd';
1534         my $path = '';
1535         if (drive_is_cdrom($drive)) {
1536             $devicetype = 'cd';
1537         } else {
1538             if ($drive->{file} =~ m|^/|) {
1539                 $path = $drive->{file};
1540                 if (my $info = path_is_scsi($path)) {
1541                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1542                         $devicetype = 'block';
1543                     } elsif ($info->{type} == 1) { # tape
1544                         $devicetype = 'generic';
1545                     }
1546                 }
1547             } else {
1548                  $path = PVE::Storage::path($storecfg, $drive->{file});
1549             }
1550
1551             if($path =~ m/^iscsi\:\/\//){
1552                 $devicetype = 'generic';
1553             }
1554         }
1555
1556         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1557             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1558         } else {
1559             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1560         }
1561
1562     } elsif ($drive->{interface} eq 'ide'){
1563         $maxdev = 2;
1564         my $controller = int($drive->{index} / $maxdev);
1565         my $unit = $drive->{index} % $maxdev;
1566         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1567
1568         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1569         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1570             $model = URI::Escape::uri_unescape($model);
1571             $device .= ",model=$model";
1572         }
1573     } elsif ($drive->{interface} eq 'sata'){
1574         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1575         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1576         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1577     } elsif ($drive->{interface} eq 'usb') {
1578         die "implement me";
1579         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1580     } else {
1581         die "unsupported interface type";
1582     }
1583
1584     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1585
1586     return $device;
1587 }
1588
1589 sub get_initiator_name {
1590     my $initiator;
1591
1592     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1593     while (defined(my $line = <$fh>)) {
1594         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1595         $initiator = $1;
1596         last;
1597     }
1598     $fh->close();
1599
1600     return $initiator;
1601 }
1602
1603 sub print_drive_full {
1604     my ($storecfg, $vmid, $drive) = @_;
1605
1606     my $path;
1607     my $volid = $drive->{file};
1608     my $format;
1609
1610     if (drive_is_cdrom($drive)) {
1611         $path = get_iso_path($storecfg, $vmid, $volid);
1612     } else {
1613         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1614         if ($storeid) {
1615             $path = PVE::Storage::path($storecfg, $volid);
1616             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1617             $format = qemu_img_format($scfg, $volname);
1618         } else {
1619             $path = $volid;
1620             $format = "raw";
1621         }
1622    }
1623
1624     my $opts = '';
1625     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1626     foreach my $o (@qemu_drive_options) {
1627         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1628     }
1629
1630     # snapshot only accepts on|off
1631     if (defined($drive->{snapshot})) {
1632         my $v = $drive->{snapshot} ? 'on' : 'off';
1633         $opts .= ",snapshot=$v";
1634     }
1635
1636     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1637         my ($dir, $qmpname) = @$type;
1638         if (my $v = $drive->{"mbps$dir"}) {
1639             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1640         }
1641         if (my $v = $drive->{"mbps${dir}_max"}) {
1642             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1643         }
1644         if (my $v = $drive->{"bps${dir}_max_length"}) {
1645             $opts .= ",throttling.bps$qmpname-max-length=$v";
1646         }
1647         if (my $v = $drive->{"iops${dir}"}) {
1648             $opts .= ",throttling.iops$qmpname=$v";
1649         }
1650         if (my $v = $drive->{"iops${dir}_max"}) {
1651             $opts .= ",throttling.iops$qmpname-max=$v";
1652         }
1653         if (my $v = $drive->{"iops${dir}_max_length"}) {
1654             $opts .= ",throttling.iops$qmpname-max-length=$v";
1655         }
1656     }
1657
1658     if (my $serial = $drive->{serial}) {
1659         $serial = URI::Escape::uri_unescape($serial);
1660         $opts .= ",serial=$serial";
1661     }
1662
1663     $opts .= ",format=$format" if $format && !$drive->{format};
1664
1665     my $cache_direct = 0;
1666
1667     if (my $cache = $drive->{cache}) {
1668         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1669     } elsif (!drive_is_cdrom($drive)) {
1670         $opts .= ",cache=none";
1671         $cache_direct = 1;
1672     }
1673
1674     # aio native works only with O_DIRECT
1675     if (!$drive->{aio}) {
1676         if($cache_direct) {
1677             $opts .= ",aio=native";
1678         } else {
1679             $opts .= ",aio=threads";
1680         }
1681     }
1682
1683     if (!drive_is_cdrom($drive)) {
1684         my $detectzeroes;
1685         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1686             $detectzeroes = 'off';
1687         } elsif ($drive->{discard}) {
1688             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1689         } else {
1690             # This used to be our default with discard not being specified:
1691             $detectzeroes = 'on';
1692         }
1693         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1694     }
1695
1696     my $pathinfo = $path ? "file=$path," : '';
1697
1698     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1699 }
1700
1701 sub print_netdevice_full {
1702     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1703
1704     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1705
1706     my $device = $net->{model};
1707     if ($net->{model} eq 'virtio') {
1708          $device = 'virtio-net-pci';
1709      };
1710
1711     my $pciaddr = print_pci_addr("$netid", $bridges);
1712     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1713     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1714         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1715         my $vectors = $net->{queues} * 2 + 2;
1716         $tmpstr .= ",vectors=$vectors,mq=on";
1717     }
1718     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1719
1720     if ($use_old_bios_files) {
1721         my $romfile;
1722         if ($device eq 'virtio-net-pci') {
1723             $romfile = 'pxe-virtio.rom';
1724         } elsif ($device eq 'e1000') {
1725             $romfile = 'pxe-e1000.rom';
1726         } elsif ($device eq 'ne2k') {
1727             $romfile = 'pxe-ne2k_pci.rom';
1728         } elsif ($device eq 'pcnet') {
1729             $romfile = 'pxe-pcnet.rom';
1730         } elsif ($device eq 'rtl8139') {
1731             $romfile = 'pxe-rtl8139.rom';
1732         }
1733         $tmpstr .= ",romfile=$romfile" if $romfile;
1734     }
1735
1736     return $tmpstr;
1737 }
1738
1739 sub print_netdev_full {
1740     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1741
1742     my $i = '';
1743     if ($netid =~ m/^net(\d+)$/) {
1744         $i = int($1);
1745     }
1746
1747     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1748
1749     my $ifname = "tap${vmid}i$i";
1750
1751     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1752     die "interface name '$ifname' is too long (max 15 character)\n"
1753         if length($ifname) >= 16;
1754
1755     my $vhostparam = '';
1756     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1757
1758     my $vmname = $conf->{name} || "vm$vmid";
1759
1760     my $netdev = "";
1761     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1762
1763     if ($net->{bridge}) {
1764         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1765     } else {
1766         $netdev = "type=user,id=$netid,hostname=$vmname";
1767     }
1768
1769     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1770
1771     return $netdev;
1772 }
1773
1774
1775 sub print_cpu_device {
1776     my ($conf, $id) = @_;
1777
1778     my $kvm = $conf->{kvm} // 1;
1779     my $cpu = $kvm ? "kvm64" : "qemu64";
1780     if (my $cputype = $conf->{cpu}) {
1781         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1782             or die "Cannot parse cpu description: $cputype\n";
1783         $cpu = $cpuconf->{cputype};
1784     }
1785
1786     my $cores = $conf->{cores} || 1;
1787
1788     my $current_core = ($id - 1) % $cores;
1789     my $current_socket = int(($id - 1 - $current_core)/$cores);
1790
1791     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1792 }
1793
1794 sub drive_is_cdrom {
1795     my ($drive) = @_;
1796
1797     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1798
1799 }
1800
1801 sub parse_number_sets {
1802     my ($set) = @_;
1803     my $res = [];
1804     foreach my $part (split(/;/, $set)) {
1805         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1806             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1807             push @$res, [ $1, $2 ];
1808         } else {
1809             die "invalid range: $part\n";
1810         }
1811     }
1812     return $res;
1813 }
1814
1815 sub parse_numa {
1816     my ($data) = @_;
1817
1818     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1819     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1820     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1821     return $res;
1822 }
1823
1824 sub parse_hostpci {
1825     my ($value) = @_;
1826
1827     return undef if !$value;
1828
1829     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1830
1831     my @idlist = split(/;/, $res->{host});
1832     delete $res->{host};
1833     foreach my $id (@idlist) {
1834         if ($id =~ /^$PCIRE$/) {
1835             if (defined($2)) {
1836                 push @{$res->{pciid}}, { id => $1, function => $2 };
1837             } else {
1838                 my $pcidevices = lspci($1);
1839                 $res->{pciid} = $pcidevices->{$1};
1840             }
1841         } else {
1842             # should have been caught by parse_property_string already
1843             die "failed to parse PCI id: $id\n";
1844         }
1845     }
1846     return $res;
1847 }
1848
1849 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1850 sub parse_net {
1851     my ($data) = @_;
1852
1853     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1854     if ($@) {
1855         warn $@;
1856         return undef;
1857     }
1858     if (!defined($res->{macaddr})) {
1859         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1860         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1861     }
1862     return $res;
1863 }
1864
1865 sub print_net {
1866     my $net = shift;
1867
1868     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1869 }
1870
1871 sub add_random_macs {
1872     my ($settings) = @_;
1873
1874     foreach my $opt (keys %$settings) {
1875         next if $opt !~ m/^net(\d+)$/;
1876         my $net = parse_net($settings->{$opt});
1877         next if !$net;
1878         $settings->{$opt} = print_net($net);
1879     }
1880 }
1881
1882 sub vm_is_volid_owner {
1883     my ($storecfg, $vmid, $volid) = @_;
1884
1885     if ($volid !~  m|^/|) {
1886         my ($path, $owner);
1887         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1888         if ($owner && ($owner == $vmid)) {
1889             return 1;
1890         }
1891     }
1892
1893     return undef;
1894 }
1895
1896 sub split_flagged_list {
1897     my $text = shift || '';
1898     $text =~ s/[,;]/ /g;
1899     $text =~ s/^\s+//;
1900     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1901 }
1902
1903 sub join_flagged_list {
1904     my ($how, $lst) = @_;
1905     join $how, map { $lst->{$_} . $_ } keys %$lst;
1906 }
1907
1908 sub vmconfig_delete_pending_option {
1909     my ($conf, $key, $force) = @_;
1910
1911     delete $conf->{pending}->{$key};
1912     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1913     $pending_delete_hash->{$key} = $force ? '!' : '';
1914     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1915 }
1916
1917 sub vmconfig_undelete_pending_option {
1918     my ($conf, $key) = @_;
1919
1920     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1921     delete $pending_delete_hash->{$key};
1922
1923     if (%$pending_delete_hash) {
1924         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1925     } else {
1926         delete $conf->{pending}->{delete};
1927     }
1928 }
1929
1930 sub vmconfig_register_unused_drive {
1931     my ($storecfg, $vmid, $conf, $drive) = @_;
1932
1933     if (!drive_is_cdrom($drive)) {
1934         my $volid = $drive->{file};
1935         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1936             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1937         }
1938     }
1939 }
1940
1941 sub vmconfig_cleanup_pending {
1942     my ($conf) = @_;
1943
1944     # remove pending changes when nothing changed
1945     my $changes;
1946     foreach my $opt (keys %{$conf->{pending}}) {
1947         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1948             $changes = 1;
1949             delete $conf->{pending}->{$opt};
1950         }
1951     }
1952
1953     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1954     my $pending_delete_hash = {};
1955     while (my ($opt, $force) = each %$current_delete_hash) {
1956         if (defined($conf->{$opt})) {
1957             $pending_delete_hash->{$opt} = $force;
1958         } else {
1959             $changes = 1;
1960         }
1961     }
1962
1963     if (%$pending_delete_hash) {
1964         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1965     } else {
1966         delete $conf->{pending}->{delete};
1967     }
1968
1969     return $changes;
1970 }
1971
1972 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1973 my $smbios1_fmt = {
1974     uuid => {
1975         type => 'string',
1976         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1977         format_description => 'UUID',
1978         description => "Set SMBIOS1 UUID.",
1979         optional => 1,
1980     },
1981     version => {
1982         type => 'string',
1983         pattern => '\S+',
1984         format_description => 'string',
1985         description => "Set SMBIOS1 version.",
1986         optional => 1,
1987     },
1988     serial => {
1989         type => 'string',
1990         pattern => '\S+',
1991         format_description => 'string',
1992         description => "Set SMBIOS1 serial number.",
1993         optional => 1,
1994     },
1995     manufacturer => {
1996         type => 'string',
1997         pattern => '\S+',
1998         format_description => 'string',
1999         description => "Set SMBIOS1 manufacturer.",
2000         optional => 1,
2001     },
2002     product => {
2003         type => 'string',
2004         pattern => '\S+',
2005         format_description => 'string',
2006         description => "Set SMBIOS1 product ID.",
2007         optional => 1,
2008     },
2009     sku => {
2010         type => 'string',
2011         pattern => '\S+',
2012         format_description => 'string',
2013         description => "Set SMBIOS1 SKU string.",
2014         optional => 1,
2015     },
2016     family => {
2017         type => 'string',
2018         pattern => '\S+',
2019         format_description => 'string',
2020         description => "Set SMBIOS1 family string.",
2021         optional => 1,
2022     },
2023 };
2024
2025 sub parse_smbios1 {
2026     my ($data) = @_;
2027
2028     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2029     warn $@ if $@;
2030     return $res;
2031 }
2032
2033 sub print_smbios1 {
2034     my ($smbios1) = @_;
2035     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2036 }
2037
2038 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2039
2040 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2041 sub verify_bootdisk {
2042     my ($value, $noerr) = @_;
2043
2044     return $value if is_valid_drivename($value);
2045
2046     return undef if $noerr;
2047
2048     die "invalid boot disk '$value'\n";
2049 }
2050
2051 sub parse_watchdog {
2052     my ($value) = @_;
2053
2054     return undef if !$value;
2055
2056     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2057     warn $@ if $@;
2058     return $res;
2059 }
2060
2061 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2062 sub verify_usb_device {
2063     my ($value, $noerr) = @_;
2064
2065     return $value if parse_usb_device($value);
2066
2067     return undef if $noerr;
2068
2069     die "unable to parse usb device\n";
2070 }
2071
2072 # add JSON properties for create and set function
2073 sub json_config_properties {
2074     my $prop = shift;
2075
2076     foreach my $opt (keys %$confdesc) {
2077         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2078         $prop->{$opt} = $confdesc->{$opt};
2079     }
2080
2081     return $prop;
2082 }
2083
2084 sub check_type {
2085     my ($key, $value) = @_;
2086
2087     die "unknown setting '$key'\n" if !$confdesc->{$key};
2088
2089     my $type = $confdesc->{$key}->{type};
2090
2091     if (!defined($value)) {
2092         die "got undefined value\n";
2093     }
2094
2095     if ($value =~ m/[\n\r]/) {
2096         die "property contains a line feed\n";
2097     }
2098
2099     if ($type eq 'boolean') {
2100         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2101         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2102         die "type check ('boolean') failed - got '$value'\n";
2103     } elsif ($type eq 'integer') {
2104         return int($1) if $value =~ m/^(\d+)$/;
2105         die "type check ('integer') failed - got '$value'\n";
2106     } elsif ($type eq 'number') {
2107         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2108         die "type check ('number') failed - got '$value'\n";
2109     } elsif ($type eq 'string') {
2110         if (my $fmt = $confdesc->{$key}->{format}) {
2111             PVE::JSONSchema::check_format($fmt, $value);
2112             return $value;
2113         }
2114         $value =~ s/^\"(.*)\"$/$1/;
2115         return $value;
2116     } else {
2117         die "internal error"
2118     }
2119 }
2120
2121 sub check_iommu_support{
2122     #fixme : need to check IOMMU support
2123     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2124
2125     my $iommu=1;
2126     return $iommu;
2127
2128 }
2129
2130 sub touch_config {
2131     my ($vmid) = @_;
2132
2133     my $conf = PVE::QemuConfig->config_file($vmid);
2134     utime undef, undef, $conf;
2135 }
2136
2137 sub destroy_vm {
2138     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2139
2140     my $conffile = PVE::QemuConfig->config_file($vmid);
2141
2142     my $conf = PVE::QemuConfig->load_config($vmid);
2143
2144     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2145
2146     if ($conf->{template}) {
2147         # check if any base image is still used by a linked clone
2148         foreach_drive($conf, sub {
2149                 my ($ds, $drive) = @_;
2150
2151                 return if drive_is_cdrom($drive);
2152
2153                 my $volid = $drive->{file};
2154
2155                 return if !$volid || $volid =~ m|^/|;
2156
2157                 die "base volume '$volid' is still in use by linked cloned\n"
2158                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2159
2160         });
2161     }
2162
2163     # only remove disks owned by this VM
2164     foreach_drive($conf, sub {
2165         my ($ds, $drive) = @_;
2166
2167         return if drive_is_cdrom($drive);
2168
2169         my $volid = $drive->{file};
2170
2171         return if !$volid || $volid =~ m|^/|;
2172
2173         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2174         return if !$path || !$owner || ($owner != $vmid);
2175
2176         eval {
2177             PVE::Storage::vdisk_free($storecfg, $volid);
2178         };
2179         warn "Could not remove disk '$volid', check manually: $@" if $@;
2180
2181     });
2182
2183     if ($keep_empty_config) {
2184         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2185     } else {
2186         unlink $conffile;
2187     }
2188
2189     # also remove unused disk
2190     eval {
2191         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2192
2193         eval {
2194             PVE::Storage::foreach_volid($dl, sub {
2195                 my ($volid, $sid, $volname, $d) = @_;
2196                 PVE::Storage::vdisk_free($storecfg, $volid);
2197             });
2198         };
2199         warn $@ if $@;
2200
2201     };
2202     warn $@ if $@;
2203 }
2204
2205 sub parse_vm_config {
2206     my ($filename, $raw) = @_;
2207
2208     return undef if !defined($raw);
2209
2210     my $res = {
2211         digest => Digest::SHA::sha1_hex($raw),
2212         snapshots => {},
2213         pending => {},
2214     };
2215
2216     $filename =~ m|/qemu-server/(\d+)\.conf$|
2217         || die "got strange filename '$filename'";
2218
2219     my $vmid = $1;
2220
2221     my $conf = $res;
2222     my $descr;
2223     my $section = '';
2224
2225     my @lines = split(/\n/, $raw);
2226     foreach my $line (@lines) {
2227         next if $line =~ m/^\s*$/;
2228
2229         if ($line =~ m/^\[PENDING\]\s*$/i) {
2230             $section = 'pending';
2231             if (defined($descr)) {
2232                 $descr =~ s/\s+$//;
2233                 $conf->{description} = $descr;
2234             }
2235             $descr = undef;
2236             $conf = $res->{$section} = {};
2237             next;
2238
2239         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2240             $section = $1;
2241             if (defined($descr)) {
2242                 $descr =~ s/\s+$//;
2243                 $conf->{description} = $descr;
2244             }
2245             $descr = undef;
2246             $conf = $res->{snapshots}->{$section} = {};
2247             next;
2248         }
2249
2250         if ($line =~ m/^\#(.*)\s*$/) {
2251             $descr = '' if !defined($descr);
2252             $descr .= PVE::Tools::decode_text($1) . "\n";
2253             next;
2254         }
2255
2256         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2257             $descr = '' if !defined($descr);
2258             $descr .= PVE::Tools::decode_text($2);
2259         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2260             $conf->{snapstate} = $1;
2261         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2262             my $key = $1;
2263             my $value = $2;
2264             $conf->{$key} = $value;
2265         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2266             my $value = $1;
2267             if ($section eq 'pending') {
2268                 $conf->{delete} = $value; # we parse this later
2269             } else {
2270                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2271             }
2272         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2273             my $key = $1;
2274             my $value = $2;
2275             eval { $value = check_type($key, $value); };
2276             if ($@) {
2277                 warn "vm $vmid - unable to parse value of '$key' - $@";
2278             } else {
2279                 $key = 'ide2' if $key eq 'cdrom';
2280                 my $fmt = $confdesc->{$key}->{format};
2281                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2282                     my $v = parse_drive($key, $value);
2283                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2284                         $v->{file} = $volid;
2285                         $value = print_drive($vmid, $v);
2286                     } else {
2287                         warn "vm $vmid - unable to parse value of '$key'\n";
2288                         next;
2289                     }
2290                 }
2291
2292                 $conf->{$key} = $value;
2293             }
2294         }
2295     }
2296
2297     if (defined($descr)) {
2298         $descr =~ s/\s+$//;
2299         $conf->{description} = $descr;
2300     }
2301     delete $res->{snapstate}; # just to be sure
2302
2303     return $res;
2304 }
2305
2306 sub write_vm_config {
2307     my ($filename, $conf) = @_;
2308
2309     delete $conf->{snapstate}; # just to be sure
2310
2311     if ($conf->{cdrom}) {
2312         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2313         $conf->{ide2} = $conf->{cdrom};
2314         delete $conf->{cdrom};
2315     }
2316
2317     # we do not use 'smp' any longer
2318     if ($conf->{sockets}) {
2319         delete $conf->{smp};
2320     } elsif ($conf->{smp}) {
2321         $conf->{sockets} = $conf->{smp};
2322         delete $conf->{cores};
2323         delete $conf->{smp};
2324     }
2325
2326     my $used_volids = {};
2327
2328     my $cleanup_config = sub {
2329         my ($cref, $pending, $snapname) = @_;
2330
2331         foreach my $key (keys %$cref) {
2332             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2333                 $key eq 'snapstate' || $key eq 'pending';
2334             my $value = $cref->{$key};
2335             if ($key eq 'delete') {
2336                 die "propertry 'delete' is only allowed in [PENDING]\n"
2337                     if !$pending;
2338                 # fixme: check syntax?
2339                 next;
2340             }
2341             eval { $value = check_type($key, $value); };
2342             die "unable to parse value of '$key' - $@" if $@;
2343
2344             $cref->{$key} = $value;
2345
2346             if (!$snapname && is_valid_drivename($key)) {
2347                 my $drive = parse_drive($key, $value);
2348                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2349             }
2350         }
2351     };
2352
2353     &$cleanup_config($conf);
2354
2355     &$cleanup_config($conf->{pending}, 1);
2356
2357     foreach my $snapname (keys %{$conf->{snapshots}}) {
2358         die "internal error" if $snapname eq 'pending';
2359         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2360     }
2361
2362     # remove 'unusedX' settings if we re-add a volume
2363     foreach my $key (keys %$conf) {
2364         my $value = $conf->{$key};
2365         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2366             delete $conf->{$key};
2367         }
2368     }
2369
2370     my $generate_raw_config = sub {
2371         my ($conf, $pending) = @_;
2372
2373         my $raw = '';
2374
2375         # add description as comment to top of file
2376         if (defined(my $descr = $conf->{description})) {
2377             if ($descr) {
2378                 foreach my $cl (split(/\n/, $descr)) {
2379                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2380                 }
2381             } else {
2382                 $raw .= "#\n" if $pending;
2383             }
2384         }
2385
2386         foreach my $key (sort keys %$conf) {
2387             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2388             $raw .= "$key: $conf->{$key}\n";
2389         }
2390         return $raw;
2391     };
2392
2393     my $raw = &$generate_raw_config($conf);
2394
2395     if (scalar(keys %{$conf->{pending}})){
2396         $raw .= "\n[PENDING]\n";
2397         $raw .= &$generate_raw_config($conf->{pending}, 1);
2398     }
2399
2400     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2401         $raw .= "\n[$snapname]\n";
2402         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2403     }
2404
2405     return $raw;
2406 }
2407
2408 sub load_defaults {
2409
2410     my $res = {};
2411
2412     # we use static defaults from our JSON schema configuration
2413     foreach my $key (keys %$confdesc) {
2414         if (defined(my $default = $confdesc->{$key}->{default})) {
2415             $res->{$key} = $default;
2416         }
2417     }
2418
2419     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2420     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2421
2422     return $res;
2423 }
2424
2425 sub config_list {
2426     my $vmlist = PVE::Cluster::get_vmlist();
2427     my $res = {};
2428     return $res if !$vmlist || !$vmlist->{ids};
2429     my $ids = $vmlist->{ids};
2430
2431     foreach my $vmid (keys %$ids) {
2432         my $d = $ids->{$vmid};
2433         next if !$d->{node} || $d->{node} ne $nodename;
2434         next if !$d->{type} || $d->{type} ne 'qemu';
2435         $res->{$vmid}->{exists} = 1;
2436     }
2437     return $res;
2438 }
2439
2440 # test if VM uses local resources (to prevent migration)
2441 sub check_local_resources {
2442     my ($conf, $noerr) = @_;
2443
2444     my $loc_res = 0;
2445
2446     $loc_res = 1 if $conf->{hostusb}; # old syntax
2447     $loc_res = 1 if $conf->{hostpci}; # old syntax
2448
2449     foreach my $k (keys %$conf) {
2450         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2451         # sockets are safe: they will recreated be on the target side post-migrate
2452         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2453         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2454     }
2455
2456     die "VM uses local resources\n" if $loc_res && !$noerr;
2457
2458     return $loc_res;
2459 }
2460
2461 # check if used storages are available on all nodes (use by migrate)
2462 sub check_storage_availability {
2463     my ($storecfg, $conf, $node) = @_;
2464
2465     foreach_drive($conf, sub {
2466         my ($ds, $drive) = @_;
2467
2468         my $volid = $drive->{file};
2469         return if !$volid;
2470
2471         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2472         return if !$sid;
2473
2474         # check if storage is available on both nodes
2475         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2476         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2477    });
2478 }
2479
2480 # list nodes where all VM images are available (used by has_feature API)
2481 sub shared_nodes {
2482     my ($conf, $storecfg) = @_;
2483
2484     my $nodelist = PVE::Cluster::get_nodelist();
2485     my $nodehash = { map { $_ => 1 } @$nodelist };
2486     my $nodename = PVE::INotify::nodename();
2487
2488     foreach_drive($conf, sub {
2489         my ($ds, $drive) = @_;
2490
2491         my $volid = $drive->{file};
2492         return if !$volid;
2493
2494         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2495         if ($storeid) {
2496             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2497             if ($scfg->{disable}) {
2498                 $nodehash = {};
2499             } elsif (my $avail = $scfg->{nodes}) {
2500                 foreach my $node (keys %$nodehash) {
2501                     delete $nodehash->{$node} if !$avail->{$node};
2502                 }
2503             } elsif (!$scfg->{shared}) {
2504                 foreach my $node (keys %$nodehash) {
2505                     delete $nodehash->{$node} if $node ne $nodename
2506                 }
2507             }
2508         }
2509     });
2510
2511     return $nodehash
2512 }
2513
2514 sub check_cmdline {
2515     my ($pidfile, $pid) = @_;
2516
2517     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2518     if (defined($fh)) {
2519         my $line = <$fh>;
2520         $fh->close;
2521         return undef if !$line;
2522         my @param = split(/\0/, $line);
2523
2524         my $cmd = $param[0];
2525         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2526
2527         for (my $i = 0; $i < scalar (@param); $i++) {
2528             my $p = $param[$i];
2529             next if !$p;
2530             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2531                 my $p = $param[$i+1];
2532                 return 1 if $p && ($p eq $pidfile);
2533                 return undef;
2534             }
2535         }
2536     }
2537     return undef;
2538 }
2539
2540 sub check_running {
2541     my ($vmid, $nocheck, $node) = @_;
2542
2543     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2544
2545     die "unable to find configuration file for VM $vmid - no such machine\n"
2546         if !$nocheck && ! -f $filename;
2547
2548     my $pidfile = pidfile_name($vmid);
2549
2550     if (my $fd = IO::File->new("<$pidfile")) {
2551         my $st = stat($fd);
2552         my $line = <$fd>;
2553         close($fd);
2554
2555         my $mtime = $st->mtime;
2556         if ($mtime > time()) {
2557             warn "file '$filename' modified in future\n";
2558         }
2559
2560         if ($line =~ m/^(\d+)$/) {
2561             my $pid = $1;
2562             if (check_cmdline($pidfile, $pid)) {
2563                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2564                     return $pid;
2565                 }
2566             }
2567         }
2568     }
2569
2570     return undef;
2571 }
2572
2573 sub vzlist {
2574
2575     my $vzlist = config_list();
2576
2577     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2578
2579     while (defined(my $de = $fd->read)) {
2580         next if $de !~ m/^(\d+)\.pid$/;
2581         my $vmid = $1;
2582         next if !defined($vzlist->{$vmid});
2583         if (my $pid = check_running($vmid)) {
2584             $vzlist->{$vmid}->{pid} = $pid;
2585         }
2586     }
2587
2588     return $vzlist;
2589 }
2590
2591 sub disksize {
2592     my ($storecfg, $conf) = @_;
2593
2594     my $bootdisk = $conf->{bootdisk};
2595     return undef if !$bootdisk;
2596     return undef if !is_valid_drivename($bootdisk);
2597
2598     return undef if !$conf->{$bootdisk};
2599
2600     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2601     return undef if !defined($drive);
2602
2603     return undef if drive_is_cdrom($drive);
2604
2605     my $volid = $drive->{file};
2606     return undef if !$volid;
2607
2608     return $drive->{size};
2609 }
2610
2611 my $last_proc_pid_stat;
2612
2613 # get VM status information
2614 # This must be fast and should not block ($full == false)
2615 # We only query KVM using QMP if $full == true (this can be slow)
2616 sub vmstatus {
2617     my ($opt_vmid, $full) = @_;
2618
2619     my $res = {};
2620
2621     my $storecfg = PVE::Storage::config();
2622
2623     my $list = vzlist();
2624     my $defaults = load_defaults();
2625
2626     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2627
2628     my $cpucount = $cpuinfo->{cpus} || 1;
2629
2630     foreach my $vmid (keys %$list) {
2631         next if $opt_vmid && ($vmid ne $opt_vmid);
2632
2633         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2634         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2635
2636         my $d = {};
2637         $d->{pid} = $list->{$vmid}->{pid};
2638
2639         # fixme: better status?
2640         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2641
2642         my $size = disksize($storecfg, $conf);
2643         if (defined($size)) {
2644             $d->{disk} = 0; # no info available
2645             $d->{maxdisk} = $size;
2646         } else {
2647             $d->{disk} = 0;
2648             $d->{maxdisk} = 0;
2649         }
2650
2651         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2652             * ($conf->{cores} || $defaults->{cores});
2653         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2654         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2655
2656         $d->{name} = $conf->{name} || "VM $vmid";
2657         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2658             : $defaults->{memory}*(1024*1024);
2659
2660         if ($conf->{balloon}) {
2661             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2662             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2663                 : $defaults->{shares};
2664         }
2665
2666         $d->{uptime} = 0;
2667         $d->{cpu} = 0;
2668         $d->{mem} = 0;
2669
2670         $d->{netout} = 0;
2671         $d->{netin} = 0;
2672
2673         $d->{diskread} = 0;
2674         $d->{diskwrite} = 0;
2675
2676         $d->{template} = PVE::QemuConfig->is_template($conf);
2677
2678         $d->{serial} = 1 if conf_has_serial($conf);
2679
2680         $res->{$vmid} = $d;
2681     }
2682
2683     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2684     foreach my $dev (keys %$netdev) {
2685         next if $dev !~ m/^tap([1-9]\d*)i/;
2686         my $vmid = $1;
2687         my $d = $res->{$vmid};
2688         next if !$d;
2689
2690         $d->{netout} += $netdev->{$dev}->{receive};
2691         $d->{netin} += $netdev->{$dev}->{transmit};
2692
2693         if ($full) {
2694             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2695             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2696         }
2697
2698     }
2699
2700     my $ctime = gettimeofday;
2701
2702     foreach my $vmid (keys %$list) {
2703
2704         my $d = $res->{$vmid};
2705         my $pid = $d->{pid};
2706         next if !$pid;
2707
2708         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2709         next if !$pstat; # not running
2710
2711         my $used = $pstat->{utime} + $pstat->{stime};
2712
2713         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2714
2715         if ($pstat->{vsize}) {
2716             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2717         }
2718
2719         my $old = $last_proc_pid_stat->{$pid};
2720         if (!$old) {
2721             $last_proc_pid_stat->{$pid} = {
2722                 time => $ctime,
2723                 used => $used,
2724                 cpu => 0,
2725             };
2726             next;
2727         }
2728
2729         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2730
2731         if ($dtime > 1000) {
2732             my $dutime = $used -  $old->{used};
2733
2734             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2735             $last_proc_pid_stat->{$pid} = {
2736                 time => $ctime,
2737                 used => $used,
2738                 cpu => $d->{cpu},
2739             };
2740         } else {
2741             $d->{cpu} = $old->{cpu};
2742         }
2743     }
2744
2745     return $res if !$full;
2746
2747     my $qmpclient = PVE::QMPClient->new();
2748
2749     my $ballooncb = sub {
2750         my ($vmid, $resp) = @_;
2751
2752         my $info = $resp->{'return'};
2753         return if !$info->{max_mem};
2754
2755         my $d = $res->{$vmid};
2756
2757         # use memory assigned to VM
2758         $d->{maxmem} = $info->{max_mem};
2759         $d->{balloon} = $info->{actual};
2760
2761         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2762             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2763             $d->{freemem} = $info->{free_mem};
2764         }
2765
2766         $d->{ballooninfo} = $info;
2767     };
2768
2769     my $blockstatscb = sub {
2770         my ($vmid, $resp) = @_;
2771         my $data = $resp->{'return'} || [];
2772         my $totalrdbytes = 0;
2773         my $totalwrbytes = 0;
2774
2775         for my $blockstat (@$data) {
2776             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2777             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2778
2779             $blockstat->{device} =~ s/drive-//;
2780             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2781         }
2782         $res->{$vmid}->{diskread} = $totalrdbytes;
2783         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2784     };
2785
2786     my $statuscb = sub {
2787         my ($vmid, $resp) = @_;
2788
2789         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2790         # this fails if ballon driver is not loaded, so this must be
2791         # the last commnand (following command are aborted if this fails).
2792         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2793
2794         my $status = 'unknown';
2795         if (!defined($status = $resp->{'return'}->{status})) {
2796             warn "unable to get VM status\n";
2797             return;
2798         }
2799
2800         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2801     };
2802
2803     foreach my $vmid (keys %$list) {
2804         next if $opt_vmid && ($vmid ne $opt_vmid);
2805         next if !$res->{$vmid}->{pid}; # not running
2806         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2807     }
2808
2809     $qmpclient->queue_execute(undef, 2);
2810
2811     foreach my $vmid (keys %$list) {
2812         next if $opt_vmid && ($vmid ne $opt_vmid);
2813         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2814     }
2815
2816     return $res;
2817 }
2818
2819 sub foreach_drive {
2820     my ($conf, $func, @param) = @_;
2821
2822     foreach my $ds (valid_drive_names()) {
2823         next if !defined($conf->{$ds});
2824
2825         my $drive = parse_drive($ds, $conf->{$ds});
2826         next if !$drive;
2827
2828         &$func($ds, $drive, @param);
2829     }
2830 }
2831
2832 sub foreach_volid {
2833     my ($conf, $func, @param) = @_;
2834
2835     my $volhash = {};
2836
2837     my $test_volid = sub {
2838         my ($volid, $is_cdrom, $replicate, $snapname) = @_;
2839
2840         return if !$volid;
2841
2842         $volhash->{$volid}->{cdrom} //= 1;
2843         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2844
2845         $volhash->{$volid}->{replicate} //= 0;
2846         $volhash->{$volid}->{replicate} = 1 if $replicate;
2847
2848         $volhash->{$volid}->{referenced_in_config} //= 0;
2849         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
2850
2851         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
2852             if defined($snapname);
2853     };
2854
2855     foreach_drive($conf, sub {
2856         my ($ds, $drive) = @_;
2857         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, undef);
2858     });
2859
2860     foreach my $snapname (keys %{$conf->{snapshots}}) {
2861         my $snap = $conf->{snapshots}->{$snapname};
2862         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
2863         foreach_drive($snap, sub {
2864             my ($ds, $drive) = @_;
2865             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $snapname);
2866         });
2867     }
2868
2869     foreach my $volid (keys %$volhash) {
2870         &$func($volid, $volhash->{$volid}, @param);
2871     }
2872 }
2873
2874 sub conf_has_serial {
2875     my ($conf) = @_;
2876
2877     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2878         if ($conf->{"serial$i"}) {
2879             return 1;
2880         }
2881     }
2882
2883     return 0;
2884 }
2885
2886 sub vga_conf_has_spice {
2887     my ($vga) = @_;
2888
2889     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2890
2891     return $1 || 1;
2892 }
2893
2894 sub config_to_command {
2895     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2896
2897     my $cmd = [];
2898     my $globalFlags = [];
2899     my $machineFlags = [];
2900     my $rtcFlags = [];
2901     my $cpuFlags = [];
2902     my $devices = [];
2903     my $pciaddr = '';
2904     my $bridges = {};
2905     my $kvmver = kvm_user_version();
2906     my $vernum = 0; # unknown
2907     my $ostype = $conf->{ostype};
2908     my $winversion = windows_version($ostype);
2909     my $kvm = $conf->{kvm} // 1;
2910
2911     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
2912
2913     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2914         $vernum = $1*1000000+$2*1000;
2915     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2916         $vernum = $1*1000000+$2*1000+$3;
2917     }
2918
2919     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2920
2921     my $have_ovz = -f '/proc/vz/vestat';
2922
2923     my $q35 = machine_type_is_q35($conf);
2924     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2925     my $machine_type = $forcemachine || $conf->{machine};
2926     my $use_old_bios_files = undef;
2927     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2928
2929     my $cpuunits = defined($conf->{cpuunits}) ?
2930             $conf->{cpuunits} : $defaults->{cpuunits};
2931
2932     push @$cmd, '/usr/bin/kvm';
2933
2934     push @$cmd, '-id', $vmid;
2935
2936     my $use_virtio = 0;
2937
2938     my $qmpsocket = qmp_socket($vmid);
2939     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2940     push @$cmd, '-mon', "chardev=qmp,mode=control";
2941
2942
2943     push @$cmd, '-pidfile' , pidfile_name($vmid);
2944
2945     push @$cmd, '-daemonize';
2946
2947     if ($conf->{smbios1}) {
2948         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2949     }
2950
2951     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2952         die "uefi base image not found\n" if ! -f $OVMF_CODE;
2953
2954         my $path;
2955         my $format;
2956         if (my $efidisk = $conf->{efidisk0}) {
2957             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
2958             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2959             $format = $d->{format};
2960             if ($storeid) {
2961                 $path = PVE::Storage::path($storecfg, $d->{file});
2962                 if (!defined($format)) {
2963                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2964                     $format = qemu_img_format($scfg, $volname);
2965                 }
2966             } else {
2967                 $path = $d->{file};
2968                 die "efidisk format must be specified\n"
2969                     if !defined($format);
2970             }
2971         } else {
2972             warn "no efidisk configured! Using temporary efivars disk.\n";
2973             $path = "/tmp/$vmid-ovmf.fd";
2974             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
2975             $format = 'raw';
2976         }
2977
2978         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
2979         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
2980     }
2981
2982
2983     # add usb controllers
2984     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2985     push @$devices, @usbcontrollers if @usbcontrollers;
2986     my $vga = $conf->{vga};
2987
2988     my $qxlnum = vga_conf_has_spice($vga);
2989     $vga = 'qxl' if $qxlnum;
2990
2991     if (!$vga) {
2992         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
2993             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
2994         } else {
2995             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
2996         }
2997     }
2998
2999     # enable absolute mouse coordinates (needed by vnc)
3000     my $tablet;
3001     if (defined($conf->{tablet})) {
3002         $tablet = $conf->{tablet};
3003     } else {
3004         $tablet = $defaults->{tablet};
3005         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3006         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3007     }
3008
3009     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3010
3011     my $kvm_off = 0;
3012     my $gpu_passthrough;
3013
3014     # host pci devices
3015     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3016         my $d = parse_hostpci($conf->{"hostpci$i"});
3017         next if !$d;
3018
3019         my $pcie = $d->{pcie};
3020         if($pcie){
3021             die "q35 machine model is not enabled" if !$q35;
3022             $pciaddr = print_pcie_addr("hostpci$i");
3023         }else{
3024             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3025         }
3026
3027         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3028         my $romfile = $d->{romfile};
3029
3030         my $xvga = '';
3031         if ($d->{'x-vga'}) {
3032             $xvga = ',x-vga=on';
3033             $kvm_off = 1;
3034             $vga = 'none';
3035             $gpu_passthrough = 1;
3036
3037             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3038                 $xvga = "";
3039             }
3040         }
3041         my $pcidevices = $d->{pciid};
3042         my $multifunction = 1 if @$pcidevices > 1;
3043
3044         my $j=0;
3045         foreach my $pcidevice (@$pcidevices) {
3046
3047             my $id = "hostpci$i";
3048             $id .= ".$j" if $multifunction;
3049             my $addr = $pciaddr;
3050             $addr .= ".$j" if $multifunction;
3051             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3052
3053             if($j == 0){
3054                 $devicestr .= "$rombar$xvga";
3055                 $devicestr .= ",multifunction=on" if $multifunction;
3056                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3057             }
3058
3059             push @$devices, '-device', $devicestr;
3060             $j++;
3061         }
3062     }
3063
3064     # usb devices
3065     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3066     push @$devices, @usbdevices if @usbdevices;
3067     # serial devices
3068     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3069         if (my $path = $conf->{"serial$i"}) {
3070             if ($path eq 'socket') {
3071                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3072                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3073                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3074             } else {
3075                 die "no such serial device\n" if ! -c $path;
3076                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3077                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3078             }
3079         }
3080     }
3081
3082     # parallel devices
3083     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3084         if (my $path = $conf->{"parallel$i"}) {
3085             die "no such parallel device\n" if ! -c $path;
3086             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3087             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3088             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3089         }
3090     }
3091
3092     my $vmname = $conf->{name} || "vm$vmid";
3093
3094     push @$cmd, '-name', $vmname;
3095
3096     my $sockets = 1;
3097     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3098     $sockets = $conf->{sockets} if  $conf->{sockets};
3099
3100     my $cores = $conf->{cores} || 1;
3101
3102     my $maxcpus = $sockets * $cores;
3103
3104     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3105
3106     my $allowed_vcpus = $cpuinfo->{cpus};
3107
3108     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3109         if ($allowed_vcpus < $maxcpus);
3110
3111     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3112
3113         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3114         for (my $i = 2; $i <= $vcpus; $i++)  {
3115             my $cpustr = print_cpu_device($conf,$i);
3116             push @$cmd, '-device', $cpustr;
3117         }
3118
3119     } else {
3120
3121         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3122     }
3123     push @$cmd, '-nodefaults';
3124
3125     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3126
3127     my $bootindex_hash = {};
3128     my $i = 1;
3129     foreach my $o (split(//, $bootorder)) {
3130         $bootindex_hash->{$o} = $i*100;
3131         $i++;
3132     }
3133
3134     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3135
3136     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3137
3138     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3139
3140     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3141
3142     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3143         my $socket = vnc_socket($vmid);
3144         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3145     } else {
3146         push @$cmd, '-nographic';
3147     }
3148
3149     # time drift fix
3150     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3151
3152     my $useLocaltime = $conf->{localtime};
3153
3154     if ($winversion >= 5) { # windows
3155         $useLocaltime = 1 if !defined($conf->{localtime});
3156
3157         # use time drift fix when acpi is enabled
3158         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3159             $tdf = 1 if !defined($conf->{tdf});
3160         }
3161     }
3162
3163     if ($winversion >= 6) {
3164         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3165         push @$cmd, '-no-hpet';
3166     }
3167
3168     push @$rtcFlags, 'driftfix=slew' if $tdf;
3169
3170     if (!$kvm) {
3171         push @$machineFlags, 'accel=tcg';
3172     }
3173
3174     if ($machine_type) {
3175         push @$machineFlags, "type=${machine_type}";
3176     }
3177
3178     if ($conf->{startdate}) {
3179         push @$rtcFlags, "base=$conf->{startdate}";
3180     } elsif ($useLocaltime) {
3181         push @$rtcFlags, 'base=localtime';
3182     }
3183
3184     my $cpu = $kvm ? "kvm64" : "qemu64";
3185     if (my $cputype = $conf->{cpu}) {
3186         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3187             or die "Cannot parse cpu description: $cputype\n";
3188         $cpu = $cpuconf->{cputype};
3189         $kvm_off = 1 if $cpuconf->{hidden};
3190
3191         if (defined(my $flags = $cpuconf->{flags})) {
3192             push @$cpuFlags, split(";", $flags);
3193         }
3194     }
3195
3196     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3197
3198     push @$cpuFlags , '-x2apic'
3199         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3200
3201     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3202
3203     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3204
3205     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3206
3207         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3208         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3209     }
3210
3211     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3212
3213     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3214
3215     push @$cpuFlags, 'kvm=off' if $kvm_off;
3216
3217     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3218         die "internal error"; # should not happen
3219
3220     push @$cpuFlags, "vendor=${cpu_vendor}"
3221         if $cpu_vendor ne 'default';
3222
3223     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3224
3225     push @$cmd, '-cpu', $cpu;
3226
3227     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3228
3229     push @$cmd, '-S' if $conf->{freeze};
3230
3231     # set keyboard layout
3232     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3233     push @$cmd, '-k', $kb if $kb;
3234
3235     # enable sound
3236     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3237     #push @$cmd, '-soundhw', 'es1370';
3238     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3239
3240     if($conf->{agent}) {
3241         my $qgasocket = qmp_socket($vmid, 1);
3242         my $pciaddr = print_pci_addr("qga0", $bridges);
3243         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3244         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3245         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3246     }
3247
3248     my $spice_port;
3249
3250     if ($qxlnum) {
3251         if ($qxlnum > 1) {
3252             if ($winversion){
3253                 for(my $i = 1; $i < $qxlnum; $i++){
3254                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3255                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3256                 }
3257             } else {
3258                 # assume other OS works like Linux
3259                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3260                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3261             }
3262         }
3263
3264         my $pciaddr = print_pci_addr("spice", $bridges);
3265
3266         my $nodename = PVE::INotify::nodename();
3267         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3268         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3269         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3270         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3271         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3272
3273         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3274
3275         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3276         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3277         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3278     }
3279
3280     # enable balloon by default, unless explicitly disabled
3281     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3282         $pciaddr = print_pci_addr("balloon0", $bridges);
3283         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3284     }
3285
3286     if ($conf->{watchdog}) {
3287         my $wdopts = parse_watchdog($conf->{watchdog});
3288         $pciaddr = print_pci_addr("watchdog", $bridges);
3289         my $watchdog = $wdopts->{model} || 'i6300esb';
3290         push @$devices, '-device', "$watchdog$pciaddr";
3291         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3292     }
3293
3294     my $vollist = [];
3295     my $scsicontroller = {};
3296     my $ahcicontroller = {};
3297     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3298
3299     # Add iscsi initiator name if available
3300     if (my $initiator = get_initiator_name()) {
3301         push @$devices, '-iscsi', "initiator-name=$initiator";
3302     }
3303
3304     foreach_drive($conf, sub {
3305         my ($ds, $drive) = @_;
3306
3307         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3308             push @$vollist, $drive->{file};
3309         }
3310
3311         # ignore efidisk here, already added in bios/fw handling code above
3312         return if $drive->{interface} eq 'efidisk';
3313
3314         $use_virtio = 1 if $ds =~ m/^virtio/;
3315
3316         if (drive_is_cdrom ($drive)) {
3317             if ($bootindex_hash->{d}) {
3318                 $drive->{bootindex} = $bootindex_hash->{d};
3319                 $bootindex_hash->{d} += 1;
3320             }
3321         } else {
3322             if ($bootindex_hash->{c}) {
3323                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3324                 $bootindex_hash->{c} += 1;
3325             }
3326         }
3327
3328         if($drive->{interface} eq 'virtio'){
3329            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3330         }
3331
3332         if ($drive->{interface} eq 'scsi') {
3333
3334             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3335
3336             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3337             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3338
3339             my $iothread = '';
3340             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3341                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3342                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3343             } elsif ($drive->{iothread}) {
3344                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3345             }
3346
3347             my $queues = '';
3348             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3349                 $queues = ",num_queues=$drive->{queues}";
3350             }
3351
3352             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3353             $scsicontroller->{$controller}=1;
3354         }
3355
3356         if ($drive->{interface} eq 'sata') {
3357            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3358            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3359            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3360            $ahcicontroller->{$controller}=1;
3361         }
3362
3363         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3364         push @$devices, '-drive',$drive_cmd;
3365         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3366     });
3367
3368     for (my $i = 0; $i < $MAX_NETS; $i++) {
3369          next if !$conf->{"net$i"};
3370          my $d = parse_net($conf->{"net$i"});
3371          next if !$d;
3372
3373          $use_virtio = 1 if $d->{model} eq 'virtio';
3374
3375          if ($bootindex_hash->{n}) {
3376             $d->{bootindex} = $bootindex_hash->{n};
3377             $bootindex_hash->{n} += 1;
3378          }
3379
3380          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3381          push @$devices, '-netdev', $netdevfull;
3382
3383          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3384          push @$devices, '-device', $netdevicefull;
3385     }
3386
3387     if (!$q35) {
3388         # add pci bridges
3389         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3390            $bridges->{1} = 1;
3391            $bridges->{2} = 1;
3392         }
3393
3394         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3395
3396         while (my ($k, $v) = each %$bridges) {
3397             $pciaddr = print_pci_addr("pci.$k");
3398             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3399         }
3400     }
3401
3402     # add custom args
3403     if ($conf->{args}) {
3404         my $aa = PVE::Tools::split_args($conf->{args});
3405         push @$cmd, @$aa;
3406     }
3407
3408     push @$cmd, @$devices;
3409     push @$cmd, '-rtc', join(',', @$rtcFlags)
3410         if scalar(@$rtcFlags);
3411     push @$cmd, '-machine', join(',', @$machineFlags)
3412         if scalar(@$machineFlags);
3413     push @$cmd, '-global', join(',', @$globalFlags)
3414         if scalar(@$globalFlags);
3415
3416     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3417 }
3418
3419 sub vnc_socket {
3420     my ($vmid) = @_;
3421     return "${var_run_tmpdir}/$vmid.vnc";
3422 }
3423
3424 sub spice_port {
3425     my ($vmid) = @_;
3426
3427     my $res = vm_mon_cmd($vmid, 'query-spice');
3428
3429     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3430 }
3431
3432 sub qmp_socket {
3433     my ($vmid, $qga) = @_;
3434     my $sockettype = $qga ? 'qga' : 'qmp';
3435     return "${var_run_tmpdir}/$vmid.$sockettype";
3436 }
3437
3438 sub pidfile_name {
3439     my ($vmid) = @_;
3440     return "${var_run_tmpdir}/$vmid.pid";
3441 }
3442
3443 sub vm_devices_list {
3444     my ($vmid) = @_;
3445
3446     my $res = vm_mon_cmd($vmid, 'query-pci');
3447     my $devices = {};
3448     foreach my $pcibus (@$res) {
3449         foreach my $device (@{$pcibus->{devices}}) {
3450             next if !$device->{'qdev_id'};
3451             if ($device->{'pci_bridge'}) {
3452                 $devices->{$device->{'qdev_id'}} = 1;
3453                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3454                     next if !$bridge_device->{'qdev_id'};
3455                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3456                     $devices->{$device->{'qdev_id'}}++;
3457                 }
3458             } else {
3459                 $devices->{$device->{'qdev_id'}} = 1;
3460             }
3461         }
3462     }
3463
3464     my $resblock = vm_mon_cmd($vmid, 'query-block');
3465     foreach my $block (@$resblock) {
3466         if($block->{device} =~ m/^drive-(\S+)/){
3467                 $devices->{$1} = 1;
3468         }
3469     }
3470
3471     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3472     foreach my $mice (@$resmice) {
3473         if ($mice->{name} eq 'QEMU HID Tablet') {
3474             $devices->{tablet} = 1;
3475             last;
3476         }
3477     }
3478
3479     # for usb devices there is no query-usb
3480     # but we can iterate over the entries in
3481     # qom-list path=/machine/peripheral
3482     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3483     foreach my $per (@$resperipheral) {
3484         if ($per->{name} =~ m/^usb\d+$/) {
3485             $devices->{$per->{name}} = 1;
3486         }
3487     }
3488
3489     return $devices;
3490 }
3491
3492 sub vm_deviceplug {
3493     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3494
3495     my $q35 = machine_type_is_q35($conf);
3496
3497     my $devices_list = vm_devices_list($vmid);
3498     return 1 if defined($devices_list->{$deviceid});
3499
3500     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3501
3502     if ($deviceid eq 'tablet') {
3503
3504         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3505
3506     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3507
3508         die "usb hotplug currently not reliable\n";
3509         # since we can't reliably hot unplug all added usb devices
3510         # and usb passthrough disables live migration
3511         # we disable usb hotplugging for now
3512         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3513
3514     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3515
3516         qemu_iothread_add($vmid, $deviceid, $device);
3517
3518         qemu_driveadd($storecfg, $vmid, $device);
3519         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3520
3521         qemu_deviceadd($vmid, $devicefull);
3522         eval { qemu_deviceaddverify($vmid, $deviceid); };
3523         if (my $err = $@) {
3524             eval { qemu_drivedel($vmid, $deviceid); };
3525             warn $@ if $@;
3526             die $err;
3527         }
3528
3529     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3530
3531
3532         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3533         my $pciaddr = print_pci_addr($deviceid);
3534         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3535
3536         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3537
3538         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3539             qemu_iothread_add($vmid, $deviceid, $device);
3540             $devicefull .= ",iothread=iothread-$deviceid";
3541         }
3542
3543         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3544             $devicefull .= ",num_queues=$device->{queues}";
3545         }
3546
3547         qemu_deviceadd($vmid, $devicefull);
3548         qemu_deviceaddverify($vmid, $deviceid);
3549
3550     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3551
3552         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3553         qemu_driveadd($storecfg, $vmid, $device);
3554
3555         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3556         eval { qemu_deviceadd($vmid, $devicefull); };
3557         if (my $err = $@) {
3558             eval { qemu_drivedel($vmid, $deviceid); };
3559             warn $@ if $@;
3560             die $err;
3561         }
3562
3563     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3564
3565         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3566
3567         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3568         my $use_old_bios_files = undef;
3569         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3570
3571         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3572         qemu_deviceadd($vmid, $netdevicefull);
3573         eval { qemu_deviceaddverify($vmid, $deviceid); };
3574         if (my $err = $@) {
3575             eval { qemu_netdevdel($vmid, $deviceid); };
3576             warn $@ if $@;
3577             die $err;
3578         }
3579
3580     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3581
3582         my $bridgeid = $2;
3583         my $pciaddr = print_pci_addr($deviceid);
3584         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3585
3586         qemu_deviceadd($vmid, $devicefull);
3587         qemu_deviceaddverify($vmid, $deviceid);
3588
3589     } else {
3590         die "can't hotplug device '$deviceid'\n";
3591     }
3592
3593     return 1;
3594 }
3595
3596 # fixme: this should raise exceptions on error!
3597 sub vm_deviceunplug {
3598     my ($vmid, $conf, $deviceid) = @_;
3599
3600     my $devices_list = vm_devices_list($vmid);
3601     return 1 if !defined($devices_list->{$deviceid});
3602
3603     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3604
3605     if ($deviceid eq 'tablet') {
3606
3607         qemu_devicedel($vmid, $deviceid);
3608
3609     } elsif ($deviceid =~ m/^usb\d+$/) {
3610
3611         die "usb hotplug currently not reliable\n";
3612         # when unplugging usb devices this way,
3613         # there may be remaining usb controllers/hubs
3614         # so we disable it for now
3615         qemu_devicedel($vmid, $deviceid);
3616         qemu_devicedelverify($vmid, $deviceid);
3617
3618     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3619
3620         qemu_devicedel($vmid, $deviceid);
3621         qemu_devicedelverify($vmid, $deviceid);
3622         qemu_drivedel($vmid, $deviceid);
3623         qemu_iothread_del($conf, $vmid, $deviceid);
3624
3625     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3626
3627         qemu_devicedel($vmid, $deviceid);
3628         qemu_devicedelverify($vmid, $deviceid);
3629         qemu_iothread_del($conf, $vmid, $deviceid);
3630
3631     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3632
3633         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3634         my $device = parse_drive($deviceid, $conf->{$deviceid});
3635         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3636
3637         qemu_devicedel($vmid, $deviceid);
3638         qemu_drivedel($vmid, $deviceid);
3639         qemu_deletescsihw($conf, $vmid, $deviceid);
3640
3641     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3642
3643         qemu_devicedel($vmid, $deviceid);
3644         qemu_devicedelverify($vmid, $deviceid);
3645         qemu_netdevdel($vmid, $deviceid);
3646
3647     } else {
3648         die "can't unplug device '$deviceid'\n";
3649     }
3650
3651     return 1;
3652 }
3653
3654 sub qemu_deviceadd {
3655     my ($vmid, $devicefull) = @_;
3656
3657     $devicefull = "driver=".$devicefull;
3658     my %options =  split(/[=,]/, $devicefull);
3659
3660     vm_mon_cmd($vmid, "device_add" , %options);
3661 }
3662
3663 sub qemu_devicedel {
3664     my ($vmid, $deviceid) = @_;
3665
3666     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3667 }
3668
3669 sub qemu_iothread_add {
3670     my($vmid, $deviceid, $device) = @_;
3671
3672     if ($device->{iothread}) {
3673         my $iothreads = vm_iothreads_list($vmid);
3674         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3675     }
3676 }
3677
3678 sub qemu_iothread_del {
3679     my($conf, $vmid, $deviceid) = @_;
3680
3681     my $device = parse_drive($deviceid, $conf->{$deviceid});
3682     if ($device->{iothread}) {
3683         my $iothreads = vm_iothreads_list($vmid);
3684         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3685     }
3686 }
3687
3688 sub qemu_objectadd {
3689     my($vmid, $objectid, $qomtype) = @_;
3690
3691     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3692
3693     return 1;
3694 }
3695
3696 sub qemu_objectdel {
3697     my($vmid, $objectid) = @_;
3698
3699     vm_mon_cmd($vmid, "object-del", id => $objectid);
3700
3701     return 1;
3702 }
3703
3704 sub qemu_driveadd {
3705     my ($storecfg, $vmid, $device) = @_;
3706
3707     my $drive = print_drive_full($storecfg, $vmid, $device);
3708     $drive =~ s/\\/\\\\/g;
3709     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3710
3711     # If the command succeeds qemu prints: "OK"
3712     return 1 if $ret =~ m/OK/s;
3713
3714     die "adding drive failed: $ret\n";
3715 }
3716
3717 sub qemu_drivedel {
3718     my($vmid, $deviceid) = @_;
3719
3720     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3721     $ret =~ s/^\s+//;
3722
3723     return 1 if $ret eq "";
3724
3725     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3726     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3727
3728     die "deleting drive $deviceid failed : $ret\n";
3729 }
3730
3731 sub qemu_deviceaddverify {
3732     my ($vmid, $deviceid) = @_;
3733
3734     for (my $i = 0; $i <= 5; $i++) {
3735          my $devices_list = vm_devices_list($vmid);
3736          return 1 if defined($devices_list->{$deviceid});
3737          sleep 1;
3738     }
3739
3740     die "error on hotplug device '$deviceid'\n";
3741 }
3742
3743
3744 sub qemu_devicedelverify {
3745     my ($vmid, $deviceid) = @_;
3746
3747     # need to verify that the device is correctly removed as device_del
3748     # is async and empty return is not reliable
3749
3750     for (my $i = 0; $i <= 5; $i++) {
3751          my $devices_list = vm_devices_list($vmid);
3752          return 1 if !defined($devices_list->{$deviceid});
3753          sleep 1;
3754     }
3755
3756     die "error on hot-unplugging device '$deviceid'\n";
3757 }
3758
3759 sub qemu_findorcreatescsihw {
3760     my ($storecfg, $conf, $vmid, $device) = @_;
3761
3762     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3763
3764     my $scsihwid="$controller_prefix$controller";
3765     my $devices_list = vm_devices_list($vmid);
3766
3767     if(!defined($devices_list->{$scsihwid})) {
3768         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3769     }
3770
3771     return 1;
3772 }
3773
3774 sub qemu_deletescsihw {
3775     my ($conf, $vmid, $opt) = @_;
3776
3777     my $device = parse_drive($opt, $conf->{$opt});
3778
3779     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3780         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3781         return 1;
3782     }
3783
3784     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3785
3786     my $devices_list = vm_devices_list($vmid);
3787     foreach my $opt (keys %{$devices_list}) {
3788         if (PVE::QemuServer::is_valid_drivename($opt)) {
3789             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3790             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3791                 return 1;
3792             }
3793         }
3794     }
3795
3796     my $scsihwid="scsihw$controller";
3797
3798     vm_deviceunplug($vmid, $conf, $scsihwid);
3799
3800     return 1;
3801 }
3802
3803 sub qemu_add_pci_bridge {
3804     my ($storecfg, $conf, $vmid, $device) = @_;
3805
3806     my $bridges = {};
3807
3808     my $bridgeid;
3809
3810     print_pci_addr($device, $bridges);
3811
3812     while (my ($k, $v) = each %$bridges) {
3813         $bridgeid = $k;
3814     }
3815     return 1 if !defined($bridgeid) || $bridgeid < 1;
3816
3817     my $bridge = "pci.$bridgeid";
3818     my $devices_list = vm_devices_list($vmid);
3819
3820     if (!defined($devices_list->{$bridge})) {
3821         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3822     }
3823
3824     return 1;
3825 }
3826
3827 sub qemu_set_link_status {
3828     my ($vmid, $device, $up) = @_;
3829
3830     vm_mon_cmd($vmid, "set_link", name => $device,
3831                up => $up ? JSON::true : JSON::false);
3832 }
3833
3834 sub qemu_netdevadd {
3835     my ($vmid, $conf, $device, $deviceid) = @_;
3836
3837     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3838     my %options =  split(/[=,]/, $netdev);
3839
3840     vm_mon_cmd($vmid, "netdev_add",  %options);
3841     return 1;
3842 }
3843
3844 sub qemu_netdevdel {
3845     my ($vmid, $deviceid) = @_;
3846
3847     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3848 }
3849
3850 sub qemu_usb_hotplug {
3851     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3852
3853     return if !$device;
3854
3855     # remove the old one first
3856     vm_deviceunplug($vmid, $conf, $deviceid);
3857
3858     # check if xhci controller is necessary and available
3859     if ($device->{usb3}) {
3860
3861         my $devicelist = vm_devices_list($vmid);
3862
3863         if (!$devicelist->{xhci}) {
3864             my $pciaddr = print_pci_addr("xhci");
3865             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3866         }
3867     }
3868     my $d = parse_usb_device($device->{host});
3869     $d->{usb3} = $device->{usb3};
3870
3871     # add the new one
3872     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3873 }
3874
3875 sub qemu_cpu_hotplug {
3876     my ($vmid, $conf, $vcpus) = @_;
3877
3878     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3879
3880     my $sockets = 1;
3881     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3882     $sockets = $conf->{sockets} if  $conf->{sockets};
3883     my $cores = $conf->{cores} || 1;
3884     my $maxcpus = $sockets * $cores;
3885
3886     $vcpus = $maxcpus if !$vcpus;
3887
3888     die "you can't add more vcpus than maxcpus\n"
3889         if $vcpus > $maxcpus;
3890
3891     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3892
3893     if ($vcpus < $currentvcpus) {
3894
3895         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3896
3897             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3898                 qemu_devicedel($vmid, "cpu$i");
3899                 my $retry = 0;
3900                 my $currentrunningvcpus = undef;
3901                 while (1) {
3902                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3903                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3904                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3905                     $retry++;
3906                     sleep 1;
3907                 }
3908                 #update conf after each succesfull cpu unplug
3909                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3910                 PVE::QemuConfig->write_config($vmid, $conf);
3911             }
3912         } else {
3913             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3914         }
3915
3916         return;
3917     }
3918
3919     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3920     die "vcpus in running vm does not match its configuration\n"
3921         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3922
3923     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3924
3925         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3926             my $cpustr = print_cpu_device($conf, $i);
3927             qemu_deviceadd($vmid, $cpustr);
3928
3929             my $retry = 0;
3930             my $currentrunningvcpus = undef;
3931             while (1) {
3932                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3933                 last if scalar(@{$currentrunningvcpus}) == $i;
3934                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3935                 sleep 1;
3936                 $retry++;
3937             }
3938             #update conf after each succesfull cpu hotplug
3939             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3940             PVE::QemuConfig->write_config($vmid, $conf);
3941         }
3942     } else {
3943
3944         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3945             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3946         }
3947     }
3948 }
3949
3950 sub qemu_block_set_io_throttle {
3951     my ($vmid, $deviceid,
3952         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3953         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3954         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3955         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3956
3957     return if !check_running($vmid) ;
3958
3959     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3960         bps => int($bps),
3961         bps_rd => int($bps_rd),
3962         bps_wr => int($bps_wr),
3963         iops => int($iops),
3964         iops_rd => int($iops_rd),
3965         iops_wr => int($iops_wr),
3966         bps_max => int($bps_max),
3967         bps_rd_max => int($bps_rd_max),
3968         bps_wr_max => int($bps_wr_max),
3969         iops_max => int($iops_max),
3970         iops_rd_max => int($iops_rd_max),
3971         iops_wr_max => int($iops_wr_max),
3972         bps_max_length => int($bps_max_length),
3973         bps_rd_max_length => int($bps_rd_max_length),
3974         bps_wr_max_length => int($bps_wr_max_length),
3975         iops_max_length => int($iops_max_length),
3976         iops_rd_max_length => int($iops_rd_max_length),
3977         iops_wr_max_length => int($iops_wr_max_length),
3978     );
3979
3980 }
3981
3982 # old code, only used to shutdown old VM after update
3983 sub __read_avail {
3984     my ($fh, $timeout) = @_;
3985
3986     my $sel = new IO::Select;
3987     $sel->add($fh);
3988
3989     my $res = '';
3990     my $buf;
3991
3992     my @ready;
3993     while (scalar (@ready = $sel->can_read($timeout))) {
3994         my $count;
3995         if ($count = $fh->sysread($buf, 8192)) {
3996             if ($buf =~ /^(.*)\(qemu\) $/s) {
3997                 $res .= $1;
3998                 last;
3999             } else {
4000                 $res .= $buf;
4001             }
4002         } else {
4003             if (!defined($count)) {
4004                 die "$!\n";
4005             }
4006             last;
4007         }
4008     }
4009
4010     die "monitor read timeout\n" if !scalar(@ready);
4011
4012     return $res;
4013 }
4014
4015 # old code, only used to shutdown old VM after update
4016 sub vm_monitor_command {
4017     my ($vmid, $cmdstr, $nocheck) = @_;
4018
4019     my $res;
4020
4021     eval {
4022         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4023
4024         my $sname = "${var_run_tmpdir}/$vmid.mon";
4025
4026         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
4027             die "unable to connect to VM $vmid socket - $!\n";
4028
4029         my $timeout = 3;
4030
4031         # hack: migrate sometime blocks the monitor (when migrate_downtime
4032         # is set)
4033         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4034             $timeout = 60*60; # 1 hour
4035         }
4036
4037         # read banner;
4038         my $data = __read_avail($sock, $timeout);
4039
4040         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
4041             die "got unexpected qemu monitor banner\n";
4042         }
4043
4044         my $sel = new IO::Select;
4045         $sel->add($sock);
4046
4047         if (!scalar(my @ready = $sel->can_write($timeout))) {
4048             die "monitor write error - timeout";
4049         }
4050
4051         my $fullcmd = "$cmdstr\r";
4052
4053         # syslog('info', "VM $vmid monitor command: $cmdstr");
4054
4055         my $b;
4056         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
4057             die "monitor write error - $!";
4058         }
4059
4060         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4061
4062         $timeout = 20;
4063
4064         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4065             $timeout = 60*60; # 1 hour
4066         } elsif ($cmdstr =~ m/^(eject|change)/) {
4067             $timeout = 60; # note: cdrom mount command is slow
4068         }
4069         if ($res = __read_avail($sock, $timeout)) {
4070
4071             my @lines = split("\r?\n", $res);
4072
4073             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4074
4075             $res = join("\n", @lines);
4076             $res .= "\n";
4077         }
4078     };
4079
4080     my $err = $@;
4081
4082     if ($err) {
4083         syslog("err", "VM $vmid monitor command failed - $err");
4084         die $err;
4085     }
4086
4087     return $res;
4088 }
4089
4090 sub qemu_block_resize {
4091     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4092
4093     my $running = check_running($vmid);
4094
4095     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4096
4097     return if !$running;
4098
4099     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4100
4101 }
4102
4103 sub qemu_volume_snapshot {
4104     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4105
4106     my $running = check_running($vmid);
4107
4108     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4109         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4110     } else {
4111         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4112     }
4113 }
4114
4115 sub qemu_volume_snapshot_delete {
4116     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4117
4118     my $running = check_running($vmid);
4119
4120     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4121         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4122     } else {
4123         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4124     }
4125 }
4126
4127 sub set_migration_caps {
4128     my ($vmid) = @_;
4129
4130     my $cap_ref = [];
4131
4132     my $enabled_cap = {
4133         "auto-converge" => 1,
4134         "xbzrle" => 1,
4135         "x-rdma-pin-all" => 0,
4136         "zero-blocks" => 0,
4137         "compress" => 0
4138     };
4139
4140     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4141
4142     for my $supported_capability (@$supported_capabilities) {
4143         push @$cap_ref, {
4144             capability => $supported_capability->{capability},
4145             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4146         };
4147     }
4148
4149     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4150 }
4151
4152 my $fast_plug_option = {
4153     'lock' => 1,
4154     'name' => 1,
4155     'onboot' => 1,
4156     'shares' => 1,
4157     'startup' => 1,
4158     'description' => 1,
4159     'protection' => 1,
4160     'vmstatestorage' => 1,
4161 };
4162
4163 # hotplug changes in [PENDING]
4164 # $selection hash can be used to only apply specified options, for
4165 # example: { cores => 1 } (only apply changed 'cores')
4166 # $errors ref is used to return error messages
4167 sub vmconfig_hotplug_pending {
4168     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4169
4170     my $defaults = load_defaults();
4171
4172     # commit values which do not have any impact on running VM first
4173     # Note: those option cannot raise errors, we we do not care about
4174     # $selection and always apply them.
4175
4176     my $add_error = sub {
4177         my ($opt, $msg) = @_;
4178         $errors->{$opt} = "hotplug problem - $msg";
4179     };
4180
4181     my $changes = 0;
4182     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4183         if ($fast_plug_option->{$opt}) {
4184             $conf->{$opt} = $conf->{pending}->{$opt};
4185             delete $conf->{pending}->{$opt};
4186             $changes = 1;
4187         }
4188     }
4189
4190     if ($changes) {
4191         PVE::QemuConfig->write_config($vmid, $conf);
4192         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4193     }
4194
4195     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4196
4197     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4198     while (my ($opt, $force) = each %$pending_delete_hash) {
4199         next if $selection && !$selection->{$opt};
4200         eval {
4201             if ($opt eq 'hotplug') {
4202                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4203             } elsif ($opt eq 'tablet') {
4204                 die "skip\n" if !$hotplug_features->{usb};
4205                 if ($defaults->{tablet}) {
4206                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4207                 } else {
4208                     vm_deviceunplug($vmid, $conf, $opt);
4209                 }
4210             } elsif ($opt =~ m/^usb\d+/) {
4211                 die "skip\n";
4212                 # since we cannot reliably hot unplug usb devices
4213                 # we are disabling it
4214                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4215                 vm_deviceunplug($vmid, $conf, $opt);
4216             } elsif ($opt eq 'vcpus') {
4217                 die "skip\n" if !$hotplug_features->{cpu};
4218                 qemu_cpu_hotplug($vmid, $conf, undef);
4219             } elsif ($opt eq 'balloon') {
4220                 # enable balloon device is not hotpluggable
4221                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4222             } elsif ($fast_plug_option->{$opt}) {
4223                 # do nothing
4224             } elsif ($opt =~ m/^net(\d+)$/) {
4225                 die "skip\n" if !$hotplug_features->{network};
4226                 vm_deviceunplug($vmid, $conf, $opt);
4227             } elsif (is_valid_drivename($opt)) {
4228                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4229                 vm_deviceunplug($vmid, $conf, $opt);
4230                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4231             } elsif ($opt =~ m/^memory$/) {
4232                 die "skip\n" if !$hotplug_features->{memory};
4233                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4234             } elsif ($opt eq 'cpuunits') {
4235                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4236             } elsif ($opt eq 'cpulimit') {
4237                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4238             } else {
4239                 die "skip\n";
4240             }
4241         };
4242         if (my $err = $@) {
4243             &$add_error($opt, $err) if $err ne "skip\n";
4244         } else {
4245             # save new config if hotplug was successful
4246             delete $conf->{$opt};
4247             vmconfig_undelete_pending_option($conf, $opt);
4248             PVE::QemuConfig->write_config($vmid, $conf);
4249             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4250         }
4251     }
4252
4253     foreach my $opt (keys %{$conf->{pending}}) {
4254         next if $selection && !$selection->{$opt};
4255         my $value = $conf->{pending}->{$opt};
4256         eval {
4257             if ($opt eq 'hotplug') {
4258                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4259             } elsif ($opt eq 'tablet') {
4260                 die "skip\n" if !$hotplug_features->{usb};
4261                 if ($value == 1) {
4262                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4263                 } elsif ($value == 0) {
4264                     vm_deviceunplug($vmid, $conf, $opt);
4265                 }
4266             } elsif ($opt =~ m/^usb\d+$/) {
4267                 die "skip\n";
4268                 # since we cannot reliably hot unplug usb devices
4269                 # we are disabling it
4270                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4271                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4272                 die "skip\n" if !$d;
4273                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4274             } elsif ($opt eq 'vcpus') {
4275                 die "skip\n" if !$hotplug_features->{cpu};
4276                 qemu_cpu_hotplug($vmid, $conf, $value);
4277             } elsif ($opt eq 'balloon') {
4278                 # enable/disable balloning device is not hotpluggable
4279                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4280                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4281                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4282
4283                 # allow manual ballooning if shares is set to zero
4284                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4285                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4286                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4287                 }
4288             } elsif ($opt =~ m/^net(\d+)$/) {
4289                 # some changes can be done without hotplug
4290                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4291                                     $vmid, $opt, $value);
4292             } elsif (is_valid_drivename($opt)) {
4293                 # some changes can be done without hotplug
4294                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4295                                      $vmid, $opt, $value, 1);
4296             } elsif ($opt =~ m/^memory$/) { #dimms
4297                 die "skip\n" if !$hotplug_features->{memory};
4298                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4299             } elsif ($opt eq 'cpuunits') {
4300                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4301             } elsif ($opt eq 'cpulimit') {
4302                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4303                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4304             } else {
4305                 die "skip\n";  # skip non-hot-pluggable options
4306             }
4307         };
4308         if (my $err = $@) {
4309             &$add_error($opt, $err) if $err ne "skip\n";
4310         } else {
4311             # save new config if hotplug was successful
4312             $conf->{$opt} = $value;
4313             delete $conf->{pending}->{$opt};
4314             PVE::QemuConfig->write_config($vmid, $conf);
4315             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4316         }
4317     }
4318 }
4319
4320 sub try_deallocate_drive {
4321     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4322
4323     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4324         my $volid = $drive->{file};
4325         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4326             my $sid = PVE::Storage::parse_volume_id($volid);
4327             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4328
4329             # check if the disk is really unused
4330             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4331                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4332             PVE::Storage::vdisk_free($storecfg, $volid);
4333             return 1;
4334         } else {
4335             # If vm is not owner of this disk remove from config
4336             return 1;
4337         }
4338     }
4339
4340     return undef;
4341 }
4342
4343 sub vmconfig_delete_or_detach_drive {
4344     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4345
4346     my $drive = parse_drive($opt, $conf->{$opt});
4347
4348     my $rpcenv = PVE::RPCEnvironment::get();
4349     my $authuser = $rpcenv->get_user();
4350
4351     if ($force) {
4352         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4353         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4354     } else {
4355         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4356     }
4357 }
4358
4359 sub vmconfig_apply_pending {
4360     my ($vmid, $conf, $storecfg) = @_;
4361
4362     # cold plug
4363
4364     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4365     while (my ($opt, $force) = each %$pending_delete_hash) {
4366         die "internal error" if $opt =~ m/^unused/;
4367         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4368         if (!defined($conf->{$opt})) {
4369             vmconfig_undelete_pending_option($conf, $opt);
4370             PVE::QemuConfig->write_config($vmid, $conf);
4371         } elsif (is_valid_drivename($opt)) {
4372             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4373             vmconfig_undelete_pending_option($conf, $opt);
4374             delete $conf->{$opt};
4375             PVE::QemuConfig->write_config($vmid, $conf);
4376         } else {
4377             vmconfig_undelete_pending_option($conf, $opt);
4378             delete $conf->{$opt};
4379             PVE::QemuConfig->write_config($vmid, $conf);
4380         }
4381     }
4382
4383     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4384
4385     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4386         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4387
4388         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4389             # skip if nothing changed
4390         } elsif (is_valid_drivename($opt)) {
4391             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4392                 if defined($conf->{$opt});
4393             $conf->{$opt} = $conf->{pending}->{$opt};
4394         } else {
4395             $conf->{$opt} = $conf->{pending}->{$opt};
4396         }
4397
4398         delete $conf->{pending}->{$opt};
4399         PVE::QemuConfig->write_config($vmid, $conf);
4400     }
4401 }
4402
4403 my $safe_num_ne = sub {
4404     my ($a, $b) = @_;
4405
4406     return 0 if !defined($a) && !defined($b);
4407     return 1 if !defined($a);
4408     return 1 if !defined($b);
4409
4410     return $a != $b;
4411 };
4412
4413 my $safe_string_ne = sub {
4414     my ($a, $b) = @_;
4415
4416     return 0 if !defined($a) && !defined($b);
4417     return 1 if !defined($a);
4418     return 1 if !defined($b);
4419
4420     return $a ne $b;
4421 };
4422
4423 sub vmconfig_update_net {
4424     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4425
4426     my $newnet = parse_net($value);
4427
4428     if ($conf->{$opt}) {
4429         my $oldnet = parse_net($conf->{$opt});
4430
4431         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4432             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4433             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4434             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4435
4436             # for non online change, we try to hot-unplug
4437             die "skip\n" if !$hotplug;
4438             vm_deviceunplug($vmid, $conf, $opt);
4439         } else {
4440
4441             die "internal error" if $opt !~ m/net(\d+)/;
4442             my $iface = "tap${vmid}i$1";
4443
4444             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4445                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4446                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4447                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4448                 PVE::Network::tap_unplug($iface);
4449                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4450             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4451                 # Rate can be applied on its own but any change above needs to
4452                 # include the rate in tap_plug since OVS resets everything.
4453                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4454             }
4455
4456             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4457                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4458             }
4459
4460             return 1;
4461         }
4462     }
4463
4464     if ($hotplug) {
4465         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4466     } else {
4467         die "skip\n";
4468     }
4469 }
4470
4471 sub vmconfig_update_disk {
4472     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4473
4474     # fixme: do we need force?
4475
4476     my $drive = parse_drive($opt, $value);
4477
4478     if ($conf->{$opt}) {
4479
4480         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4481
4482             my $media = $drive->{media} || 'disk';
4483             my $oldmedia = $old_drive->{media} || 'disk';
4484             die "unable to change media type\n" if $media ne $oldmedia;
4485
4486             if (!drive_is_cdrom($old_drive)) {
4487
4488                 if ($drive->{file} ne $old_drive->{file}) {
4489
4490                     die "skip\n" if !$hotplug;
4491
4492                     # unplug and register as unused
4493                     vm_deviceunplug($vmid, $conf, $opt);
4494                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4495
4496                 } else {
4497                     # update existing disk
4498
4499                     # skip non hotpluggable value
4500                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4501                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4502                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4503                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4504                         die "skip\n";
4505                     }
4506
4507                     # apply throttle
4508                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4509                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4510                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4511                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4512                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4513                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4514                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4515                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4516                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4517                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4518                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4519                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4520                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4521                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4522                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4523                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4524                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4525                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4526
4527                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4528                                                    ($drive->{mbps} || 0)*1024*1024,
4529                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4530                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4531                                                    $drive->{iops} || 0,
4532                                                    $drive->{iops_rd} || 0,
4533                                                    $drive->{iops_wr} || 0,
4534                                                    ($drive->{mbps_max} || 0)*1024*1024,
4535                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4536                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4537                                                    $drive->{iops_max} || 0,
4538                                                    $drive->{iops_rd_max} || 0,
4539                                                    $drive->{iops_wr_max} || 0,
4540                                                    $drive->{bps_max_length} || 1,
4541                                                    $drive->{bps_rd_max_length} || 1,
4542                                                    $drive->{bps_wr_max_length} || 1,
4543                                                    $drive->{iops_max_length} || 1,
4544                                                    $drive->{iops_rd_max_length} || 1,
4545                                                    $drive->{iops_wr_max_length} || 1);
4546
4547                     }
4548
4549                     return 1;
4550                 }
4551
4552             } else { # cdrom
4553
4554                 if ($drive->{file} eq 'none') {
4555                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4556                 } else {
4557                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4558                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4559                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4560                 }
4561
4562                 return 1;
4563             }
4564         }
4565     }
4566
4567     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4568     # hotplug new disks
4569     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4570     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4571 }
4572
4573 sub vm_start {
4574     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4575         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4576
4577     PVE::QemuConfig->lock_config($vmid, sub {
4578         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4579
4580         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4581
4582         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4583
4584         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4585
4586         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4587             vmconfig_apply_pending($vmid, $conf, $storecfg);
4588             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4589         }
4590
4591         my $defaults = load_defaults();
4592
4593         # set environment variable useful inside network script
4594         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4595
4596         my $local_volumes = {};
4597
4598         if ($targetstorage) {
4599             foreach_drive($conf, sub {
4600                 my ($ds, $drive) = @_;
4601
4602                 return if drive_is_cdrom($drive);
4603
4604                 my $volid = $drive->{file};
4605
4606                 return if !$volid;
4607
4608                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4609
4610                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4611                 return if $scfg->{shared};
4612                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4613             });
4614
4615             my $format = undef;
4616
4617             foreach my $opt (sort keys %$local_volumes) {
4618
4619                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4620                 my $drive = parse_drive($opt, $conf->{$opt});
4621
4622                 #if remote storage is specified, use default format
4623                 if ($targetstorage && $targetstorage ne "1") {
4624                     $storeid = $targetstorage;
4625                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4626                     $format = $defFormat;
4627                 } else {
4628                     #else we use same format than original
4629                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4630                     $format = qemu_img_format($scfg, $volid);
4631                 }
4632
4633                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4634                 my $newdrive = $drive;
4635                 $newdrive->{format} = $format;
4636                 $newdrive->{file} = $newvolid;
4637                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4638                 $local_volumes->{$opt} = $drivestr;
4639                 #pass drive to conf for command line
4640                 $conf->{$opt} = $drivestr;
4641             }
4642         }
4643
4644         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4645
4646         my $migrate_port = 0;
4647         my $migrate_uri;
4648         if ($statefile) {
4649             if ($statefile eq 'tcp') {
4650                 my $localip = "localhost";
4651                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4652                 my $nodename = PVE::INotify::nodename();
4653
4654                 if (!defined($migration_type)) {
4655                     if (defined($datacenterconf->{migration}->{type})) {
4656                         $migration_type = $datacenterconf->{migration}->{type};
4657                     } else {
4658                         $migration_type = 'secure';
4659                     }
4660                 }
4661
4662                 if ($migration_type eq 'insecure') {
4663                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4664                     if ($migrate_network_addr) {
4665                         $localip = $migrate_network_addr;
4666                     } else {
4667                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4668                     }
4669
4670                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4671                 }
4672
4673                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4674                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4675                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4676                 push @$cmd, '-incoming', $migrate_uri;
4677                 push @$cmd, '-S';
4678
4679             } elsif ($statefile eq 'unix') {
4680                 # should be default for secure migrations as a ssh TCP forward
4681                 # tunnel is not deterministic reliable ready and fails regurarly
4682                 # to set up in time, so use UNIX socket forwards
4683                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4684                 unlink $socket_addr;
4685
4686                 $migrate_uri = "unix:$socket_addr";
4687
4688                 push @$cmd, '-incoming', $migrate_uri;
4689                 push @$cmd, '-S';
4690
4691             } else {
4692                 push @$cmd, '-loadstate', $statefile;
4693             }
4694         } elsif ($paused) {
4695             push @$cmd, '-S';
4696         }
4697
4698         # host pci devices
4699         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4700           my $d = parse_hostpci($conf->{"hostpci$i"});
4701           next if !$d;
4702           my $pcidevices = $d->{pciid};
4703           foreach my $pcidevice (@$pcidevices) {
4704                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4705
4706                 my $info = pci_device_info("0000:$pciid");
4707                 die "IOMMU not present\n" if !check_iommu_support();
4708                 die "no pci device info for device '$pciid'\n" if !$info;
4709                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4710                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4711           }
4712         }
4713
4714         PVE::Storage::activate_volumes($storecfg, $vollist);
4715
4716         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4717             my $cmd = [];
4718             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4719             eval  { run_command($cmd); };
4720         }
4721
4722         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4723                                                   : $defaults->{cpuunits};
4724
4725         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4726         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4727
4728         my %properties = (
4729             Slice => 'qemu.slice',
4730             KillMode => 'none',
4731             CPUShares => $cpuunits
4732         );
4733
4734         if (my $cpulimit = $conf->{cpulimit}) {
4735             $properties{CPUQuota} = int($cpulimit * 100);
4736         }
4737         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4738
4739         if ($conf->{hugepages}) {
4740
4741             my $code = sub {
4742                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4743                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4744
4745                 PVE::QemuServer::Memory::hugepages_mount();
4746                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4747
4748                 eval  {
4749                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4750                     run_command($cmd, %run_params);
4751                 };
4752
4753                 if (my $err = $@) {
4754                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4755                     die $err;
4756                 }
4757
4758                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4759             };
4760             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4761
4762         } else {
4763             eval  {
4764                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4765                 run_command($cmd, %run_params);
4766             };
4767         }
4768
4769         if (my $err = $@) {
4770             # deactivate volumes if start fails
4771             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4772             die "start failed: $err";
4773         }
4774
4775         print "migration listens on $migrate_uri\n" if $migrate_uri;
4776
4777         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4778             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4779             warn $@ if $@;
4780         }
4781
4782         #start nbd server for storage migration
4783         if ($targetstorage) {
4784             my $nodename = PVE::INotify::nodename();
4785             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4786             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4787             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4788             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4789
4790             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4791
4792             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4793
4794             foreach my $opt (sort keys %$local_volumes) {
4795                 my $volid = $local_volumes->{$opt};
4796                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4797                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4798                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4799             }
4800         }
4801
4802         if ($migratedfrom) {
4803             eval {
4804                 set_migration_caps($vmid);
4805             };
4806             warn $@ if $@;
4807
4808             if ($spice_port) {
4809                 print "spice listens on port $spice_port\n";
4810                 if ($spice_ticket) {
4811                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4812                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4813                 }
4814             }
4815
4816         } else {
4817             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4818                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4819                     if $conf->{balloon};
4820             }
4821
4822             foreach my $opt (keys %$conf) {
4823                 next if $opt !~  m/^net\d+$/;
4824                 my $nicconf = parse_net($conf->{$opt});
4825                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4826             }
4827         }
4828
4829         vm_mon_cmd_nocheck($vmid, 'qom-set',
4830                     path => "machine/peripheral/balloon0",
4831                     property => "guest-stats-polling-interval",
4832                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4833
4834     });
4835 }
4836
4837 sub vm_mon_cmd {
4838     my ($vmid, $execute, %params) = @_;
4839
4840     my $cmd = { execute => $execute, arguments => \%params };
4841     vm_qmp_command($vmid, $cmd);
4842 }
4843
4844 sub vm_mon_cmd_nocheck {
4845     my ($vmid, $execute, %params) = @_;
4846
4847     my $cmd = { execute => $execute, arguments => \%params };
4848     vm_qmp_command($vmid, $cmd, 1);
4849 }
4850
4851 sub vm_qmp_command {
4852     my ($vmid, $cmd, $nocheck) = @_;
4853
4854     my $res;
4855
4856     my $timeout;
4857     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4858         $timeout = $cmd->{arguments}->{timeout};
4859         delete $cmd->{arguments}->{timeout};
4860     }
4861
4862     eval {
4863         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4864         my $sname = qmp_socket($vmid);
4865         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4866             my $qmpclient = PVE::QMPClient->new();
4867
4868             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4869         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4870             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4871                 if scalar(%{$cmd->{arguments}});
4872             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4873         } else {
4874             die "unable to open monitor socket\n";
4875         }
4876     };
4877     if (my $err = $@) {
4878         syslog("err", "VM $vmid qmp command failed - $err");
4879         die $err;
4880     }
4881
4882     return $res;
4883 }
4884
4885 sub vm_human_monitor_command {
4886     my ($vmid, $cmdline) = @_;
4887
4888     my $res;
4889
4890     my $cmd = {
4891         execute => 'human-monitor-command',
4892         arguments => { 'command-line' => $cmdline},
4893     };
4894
4895     return vm_qmp_command($vmid, $cmd);
4896 }
4897
4898 sub vm_commandline {
4899     my ($storecfg, $vmid) = @_;
4900
4901     my $conf = PVE::QemuConfig->load_config($vmid);
4902
4903     my $defaults = load_defaults();
4904
4905     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4906
4907     return PVE::Tools::cmd2string($cmd);
4908 }
4909
4910 sub vm_reset {
4911     my ($vmid, $skiplock) = @_;
4912
4913     PVE::QemuConfig->lock_config($vmid, sub {
4914
4915         my $conf = PVE::QemuConfig->load_config($vmid);
4916
4917         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4918
4919         vm_mon_cmd($vmid, "system_reset");
4920     });
4921 }
4922
4923 sub get_vm_volumes {
4924     my ($conf) = @_;
4925
4926     my $vollist = [];
4927     foreach_volid($conf, sub {
4928         my ($volid, $attr) = @_;
4929
4930         return if $volid =~ m|^/|;
4931
4932         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4933         return if !$sid;
4934
4935         push @$vollist, $volid;
4936     });
4937
4938     return $vollist;
4939 }
4940
4941 sub vm_stop_cleanup {
4942     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4943
4944     eval {
4945
4946         if (!$keepActive) {
4947             my $vollist = get_vm_volumes($conf);
4948             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4949         }
4950
4951         foreach my $ext (qw(mon qmp pid vnc qga)) {
4952             unlink "/var/run/qemu-server/${vmid}.$ext";
4953         }
4954
4955         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4956     };
4957     warn $@ if $@; # avoid errors - just warn
4958 }
4959
4960 # Note: use $nockeck to skip tests if VM configuration file exists.
4961 # We need that when migration VMs to other nodes (files already moved)
4962 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4963 sub vm_stop {
4964     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4965
4966     $force = 1 if !defined($force) && !$shutdown;
4967
4968     if ($migratedfrom){
4969         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4970         kill 15, $pid if $pid;
4971         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4972         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4973         return;
4974     }
4975
4976     PVE::QemuConfig->lock_config($vmid, sub {
4977
4978         my $pid = check_running($vmid, $nocheck);
4979         return if !$pid;
4980
4981         my $conf;
4982         if (!$nocheck) {
4983             $conf = PVE::QemuConfig->load_config($vmid);
4984             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4985             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4986                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4987                 $timeout = $opts->{down} if $opts->{down};
4988             }
4989         }
4990
4991         $timeout = 60 if !defined($timeout);
4992
4993         eval {
4994             if ($shutdown) {
4995                 if (defined($conf) && $conf->{agent}) {
4996                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4997                 } else {
4998                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4999                 }
5000             } else {
5001                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5002             }
5003         };
5004         my $err = $@;
5005
5006         if (!$err) {
5007             my $count = 0;
5008             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5009                 $count++;
5010                 sleep 1;
5011             }
5012
5013             if ($count >= $timeout) {
5014                 if ($force) {
5015                     warn "VM still running - terminating now with SIGTERM\n";
5016                     kill 15, $pid;
5017                 } else {
5018                     die "VM quit/powerdown failed - got timeout\n";
5019                 }
5020             } else {
5021                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5022                 return;
5023             }
5024         } else {
5025             if ($force) {
5026                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5027                 kill 15, $pid;
5028             } else {
5029                 die "VM quit/powerdown failed\n";
5030             }
5031         }
5032
5033         # wait again
5034         $timeout = 10;
5035
5036         my $count = 0;
5037         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5038             $count++;
5039             sleep 1;
5040         }
5041
5042         if ($count >= $timeout) {
5043             warn "VM still running - terminating now with SIGKILL\n";
5044             kill 9, $pid;
5045             sleep 1;
5046         }
5047
5048         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5049    });
5050 }
5051
5052 sub vm_suspend {
5053     my ($vmid, $skiplock) = @_;
5054
5055     PVE::QemuConfig->lock_config($vmid, sub {
5056
5057         my $conf = PVE::QemuConfig->load_config($vmid);
5058
5059         PVE::QemuConfig->check_lock($conf)
5060             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5061
5062         vm_mon_cmd($vmid, "stop");
5063     });
5064 }
5065
5066 sub vm_resume {
5067     my ($vmid, $skiplock, $nocheck) = @_;
5068
5069     PVE::QemuConfig->lock_config($vmid, sub {
5070
5071         if (!$nocheck) {
5072
5073             my $conf = PVE::QemuConfig->load_config($vmid);
5074
5075             PVE::QemuConfig->check_lock($conf)
5076                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5077
5078             vm_mon_cmd($vmid, "cont");
5079
5080         } else {
5081             vm_mon_cmd_nocheck($vmid, "cont");
5082         }
5083     });
5084 }
5085
5086 sub vm_sendkey {
5087     my ($vmid, $skiplock, $key) = @_;
5088
5089     PVE::QemuConfig->lock_config($vmid, sub {
5090
5091         my $conf = PVE::QemuConfig->load_config($vmid);
5092
5093         # there is no qmp command, so we use the human monitor command
5094         vm_human_monitor_command($vmid, "sendkey $key");
5095     });
5096 }
5097
5098 sub vm_destroy {
5099     my ($storecfg, $vmid, $skiplock) = @_;
5100
5101     PVE::QemuConfig->lock_config($vmid, sub {
5102
5103         my $conf = PVE::QemuConfig->load_config($vmid);
5104
5105         if (!check_running($vmid)) {
5106             destroy_vm($storecfg, $vmid, undef, $skiplock);
5107         } else {
5108             die "VM $vmid is running - destroy failed\n";
5109         }
5110     });
5111 }
5112
5113 # pci helpers
5114
5115 sub file_write {
5116     my ($filename, $buf) = @_;
5117
5118     my $fh = IO::File->new($filename, "w");
5119     return undef if !$fh;
5120
5121     my $res = print $fh $buf;
5122
5123     $fh->close();
5124
5125     return $res;
5126 }
5127
5128 sub pci_device_info {
5129     my ($name) = @_;
5130
5131     my $res;
5132
5133     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5134     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5135
5136     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5137     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5138
5139     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5140     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5141
5142     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5143     return undef if !defined($product) || $product !~ s/^0x//;
5144
5145     $res = {
5146         name => $name,
5147         vendor => $vendor,
5148         product => $product,
5149         domain => $domain,
5150         bus => $bus,
5151         slot => $slot,
5152         func => $func,
5153         irq => $irq,
5154         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5155     };
5156
5157     return $res;
5158 }
5159
5160 sub pci_dev_reset {
5161     my ($dev) = @_;
5162
5163     my $name = $dev->{name};
5164
5165     my $fn = "$pcisysfs/devices/$name/reset";
5166
5167     return file_write($fn, "1");
5168 }
5169
5170 sub pci_dev_bind_to_vfio {
5171     my ($dev) = @_;
5172
5173     my $name = $dev->{name};
5174
5175     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5176
5177     if (!-d $vfio_basedir) {
5178         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5179     }
5180     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5181
5182     my $testdir = "$vfio_basedir/$name";
5183     return 1 if -d $testdir;
5184
5185     my $data = "$dev->{vendor} $dev->{product}";
5186     return undef if !file_write("$vfio_basedir/new_id", $data);
5187
5188     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5189     if (!file_write($fn, $name)) {
5190         return undef if -f $fn;
5191     }
5192
5193     $fn = "$vfio_basedir/bind";
5194     if (! -d $testdir) {
5195         return undef if !file_write($fn, $name);
5196     }
5197
5198     return -d $testdir;
5199 }
5200
5201 sub pci_dev_group_bind_to_vfio {
5202     my ($pciid) = @_;
5203
5204     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5205
5206     if (!-d $vfio_basedir) {
5207         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5208     }
5209     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5210
5211     # get IOMMU group devices
5212     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5213       my @devs = grep /^0000:/, readdir($D);
5214     closedir($D);
5215
5216     foreach my $pciid (@devs) {
5217         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5218
5219         # pci bridges, switches or root ports are not supported
5220         # they have a pci_bus subdirectory so skip them
5221         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5222
5223         my $info = pci_device_info($1);
5224         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5225     }
5226
5227     return 1;
5228 }
5229
5230 # vzdump restore implementaion
5231
5232 sub tar_archive_read_firstfile {
5233     my $archive = shift;
5234
5235     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5236
5237     # try to detect archive type first
5238     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5239         die "unable to open file '$archive'\n";
5240     my $firstfile = <$fh>;
5241     kill 15, $pid;
5242     close $fh;
5243
5244     die "ERROR: archive contaions no data\n" if !$firstfile;
5245     chomp $firstfile;
5246
5247     return $firstfile;
5248 }
5249
5250 sub tar_restore_cleanup {
5251     my ($storecfg, $statfile) = @_;
5252
5253     print STDERR "starting cleanup\n";
5254
5255     if (my $fd = IO::File->new($statfile, "r")) {
5256         while (defined(my $line = <$fd>)) {
5257             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5258                 my $volid = $2;
5259                 eval {
5260                     if ($volid =~ m|^/|) {
5261                         unlink $volid || die 'unlink failed\n';
5262                     } else {
5263                         PVE::Storage::vdisk_free($storecfg, $volid);
5264                     }
5265                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5266                 };
5267                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5268             } else {
5269                 print STDERR "unable to parse line in statfile - $line";
5270             }
5271         }
5272         $fd->close();
5273     }
5274 }
5275
5276 sub restore_archive {
5277     my ($archive, $vmid, $user, $opts) = @_;
5278
5279     my $format = $opts->{format};
5280     my $comp;
5281
5282     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5283         $format = 'tar' if !$format;
5284         $comp = 'gzip';
5285     } elsif ($archive =~ m/\.tar$/) {
5286         $format = 'tar' if !$format;
5287     } elsif ($archive =~ m/.tar.lzo$/) {
5288         $format = 'tar' if !$format;
5289         $comp = 'lzop';
5290     } elsif ($archive =~ m/\.vma$/) {
5291         $format = 'vma' if !$format;
5292     } elsif ($archive =~ m/\.vma\.gz$/) {
5293         $format = 'vma' if !$format;
5294         $comp = 'gzip';
5295     } elsif ($archive =~ m/\.vma\.lzo$/) {
5296         $format = 'vma' if !$format;
5297         $comp = 'lzop';
5298     } else {
5299         $format = 'vma' if !$format; # default
5300     }
5301
5302     # try to detect archive format
5303     if ($format eq 'tar') {
5304         return restore_tar_archive($archive, $vmid, $user, $opts);
5305     } else {
5306         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5307     }
5308 }
5309
5310 sub restore_update_config_line {
5311     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5312
5313     return if $line =~ m/^\#qmdump\#/;
5314     return if $line =~ m/^\#vzdump\#/;
5315     return if $line =~ m/^lock:/;
5316     return if $line =~ m/^unused\d+:/;
5317     return if $line =~ m/^parent:/;
5318     return if $line =~ m/^template:/; # restored VM is never a template
5319
5320     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5321     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5322         # try to convert old 1.X settings
5323         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5324         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5325             my ($model, $macaddr) = split(/\=/, $devconfig);
5326             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5327             my $net = {
5328                 model => $model,
5329                 bridge => "vmbr$ind",
5330                 macaddr => $macaddr,
5331             };
5332             my $netstr = print_net($net);
5333
5334             print $outfd "net$cookie->{netcount}: $netstr\n";
5335             $cookie->{netcount}++;
5336         }
5337     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5338         my ($id, $netstr) = ($1, $2);
5339         my $net = parse_net($netstr);
5340         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5341         $netstr = print_net($net);
5342         print $outfd "$id: $netstr\n";
5343     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5344         my $virtdev = $1;
5345         my $value = $3;
5346         my $di = parse_drive($virtdev, $value);
5347         if (defined($di->{backup}) && !$di->{backup}) {
5348             print $outfd "#$line";
5349         } elsif ($map->{$virtdev}) {
5350             delete $di->{format}; # format can change on restore
5351             $di->{file} = $map->{$virtdev};
5352             $value = print_drive($vmid, $di);
5353             print $outfd "$virtdev: $value\n";
5354         } else {
5355             print $outfd $line;
5356         }
5357     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5358         my ($uuid, $uuid_str);
5359         UUID::generate($uuid);
5360         UUID::unparse($uuid, $uuid_str);
5361         my $smbios1 = parse_smbios1($2);
5362         $smbios1->{uuid} = $uuid_str;
5363         print $outfd $1.print_smbios1($smbios1)."\n";
5364     } else {
5365         print $outfd $line;
5366     }
5367 }
5368
5369 sub scan_volids {
5370     my ($cfg, $vmid) = @_;
5371
5372     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5373
5374     my $volid_hash = {};
5375     foreach my $storeid (keys %$info) {
5376         foreach my $item (@{$info->{$storeid}}) {
5377             next if !($item->{volid} && $item->{size});
5378             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5379             $volid_hash->{$item->{volid}} = $item;
5380         }
5381     }
5382
5383     return $volid_hash;
5384 }
5385
5386 sub is_volume_in_use {
5387     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5388
5389     my $path = PVE::Storage::path($storecfg, $volid);
5390
5391     my $scan_config = sub {
5392         my ($cref, $snapname) = @_;
5393
5394         foreach my $key (keys %$cref) {
5395             my $value = $cref->{$key};
5396             if (is_valid_drivename($key)) {
5397                 next if $skip_drive && $key eq $skip_drive;
5398                 my $drive = parse_drive($key, $value);
5399                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5400                 return 1 if $volid eq $drive->{file};
5401                 if ($drive->{file} =~ m!^/!) {
5402                     return 1 if $drive->{file} eq $path;
5403                 } else {
5404                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5405                     next if !$storeid;
5406                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5407                     next if !$scfg;
5408                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5409                 }
5410             }
5411         }
5412
5413         return 0;
5414     };
5415
5416     return 1 if &$scan_config($conf);
5417
5418     undef $skip_drive;
5419
5420     foreach my $snapname (keys %{$conf->{snapshots}}) {
5421         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5422     }
5423
5424     return 0;
5425 }
5426
5427 sub update_disksize {
5428     my ($vmid, $conf, $volid_hash) = @_;
5429
5430     my $changes;
5431
5432     # used and unused disks
5433     my $referenced = {};
5434
5435     # Note: it is allowed to define multiple storages with same path (alias), so
5436     # we need to check both 'volid' and real 'path' (two different volid can point
5437     # to the same path).
5438
5439     my $referencedpath = {};
5440
5441     # update size info
5442     foreach my $opt (keys %$conf) {
5443         if (is_valid_drivename($opt)) {
5444             my $drive = parse_drive($opt, $conf->{$opt});
5445             my $volid = $drive->{file};
5446             next if !$volid;
5447
5448             $referenced->{$volid} = 1;
5449             if ($volid_hash->{$volid} &&
5450                 (my $path = $volid_hash->{$volid}->{path})) {
5451                 $referencedpath->{$path} = 1;
5452             }
5453
5454             next if drive_is_cdrom($drive);
5455             next if !$volid_hash->{$volid};
5456
5457             $drive->{size} = $volid_hash->{$volid}->{size};
5458             my $new = print_drive($vmid, $drive);
5459             if ($new ne $conf->{$opt}) {
5460                 $changes = 1;
5461                 $conf->{$opt} = $new;
5462             }
5463         }
5464     }
5465
5466     # remove 'unusedX' entry if volume is used
5467     foreach my $opt (keys %$conf) {
5468         next if $opt !~ m/^unused\d+$/;
5469         my $volid = $conf->{$opt};
5470         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5471         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5472             $changes = 1;
5473             delete $conf->{$opt};
5474         }
5475
5476         $referenced->{$volid} = 1;
5477         $referencedpath->{$path} = 1 if $path;
5478     }
5479
5480     foreach my $volid (sort keys %$volid_hash) {
5481         next if $volid =~ m/vm-$vmid-state-/;
5482         next if $referenced->{$volid};
5483         my $path = $volid_hash->{$volid}->{path};
5484         next if !$path; # just to be sure
5485         next if $referencedpath->{$path};
5486         $changes = 1;
5487         PVE::QemuConfig->add_unused_volume($conf, $volid);
5488         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5489     }
5490
5491     return $changes;
5492 }
5493
5494 sub rescan {
5495     my ($vmid, $nolock) = @_;
5496
5497     my $cfg = PVE::Storage::config();
5498
5499     my $volid_hash = scan_volids($cfg, $vmid);
5500
5501     my $updatefn =  sub {
5502         my ($vmid) = @_;
5503
5504         my $conf = PVE::QemuConfig->load_config($vmid);
5505
5506         PVE::QemuConfig->check_lock($conf);
5507
5508         my $vm_volids = {};
5509         foreach my $volid (keys %$volid_hash) {
5510             my $info = $volid_hash->{$volid};
5511             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5512         }
5513
5514         my $changes = update_disksize($vmid, $conf, $vm_volids);
5515
5516         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5517     };
5518
5519     if (defined($vmid)) {
5520         if ($nolock) {
5521             &$updatefn($vmid);
5522         } else {
5523             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5524         }
5525     } else {
5526         my $vmlist = config_list();
5527         foreach my $vmid (keys %$vmlist) {
5528             if ($nolock) {
5529                 &$updatefn($vmid);
5530             } else {
5531                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5532             }
5533         }
5534     }
5535 }
5536
5537 sub restore_vma_archive {
5538     my ($archive, $vmid, $user, $opts, $comp) = @_;
5539
5540     my $input = $archive eq '-' ? "<&STDIN" : undef;
5541     my $readfrom = $archive;
5542
5543     my $uncomp = '';
5544     if ($comp) {
5545         $readfrom = '-';
5546         my $qarchive = PVE::Tools::shellquote($archive);
5547         if ($comp eq 'gzip') {
5548             $uncomp = "zcat $qarchive|";
5549         } elsif ($comp eq 'lzop') {
5550             $uncomp = "lzop -d -c $qarchive|";
5551         } else {
5552             die "unknown compression method '$comp'\n";
5553         }
5554
5555     }
5556
5557     my $tmpdir = "/var/tmp/vzdumptmp$$";
5558     rmtree $tmpdir;
5559
5560     # disable interrupts (always do cleanups)
5561     local $SIG{INT} =
5562         local $SIG{TERM} =
5563         local $SIG{QUIT} =
5564         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5565
5566     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5567     POSIX::mkfifo($mapfifo, 0600);
5568     my $fifofh;
5569
5570     my $openfifo = sub {
5571         open($fifofh, '>', $mapfifo) || die $!;
5572     };
5573
5574     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5575
5576     my $oldtimeout;
5577     my $timeout = 5;
5578
5579     my $devinfo = {};
5580
5581     my $rpcenv = PVE::RPCEnvironment::get();
5582
5583     my $conffile = PVE::QemuConfig->config_file($vmid);
5584     my $tmpfn = "$conffile.$$.tmp";
5585
5586     # Note: $oldconf is undef if VM does not exists
5587     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5588     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5589
5590     my $print_devmap = sub {
5591         my $virtdev_hash = {};
5592
5593         my $cfgfn = "$tmpdir/qemu-server.conf";
5594
5595         # we can read the config - that is already extracted
5596         my $fh = IO::File->new($cfgfn, "r") ||
5597             "unable to read qemu-server.conf - $!\n";
5598
5599         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5600         if (-f $fwcfgfn) {
5601             my $pve_firewall_dir = '/etc/pve/firewall';
5602             mkdir $pve_firewall_dir; # make sure the dir exists
5603             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5604         }
5605
5606         while (defined(my $line = <$fh>)) {
5607             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5608                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5609                 die "archive does not contain data for drive '$virtdev'\n"
5610                     if !$devinfo->{$devname};
5611                 if (defined($opts->{storage})) {
5612                     $storeid = $opts->{storage} || 'local';
5613                 } elsif (!$storeid) {
5614                     $storeid = 'local';
5615                 }
5616                 $format = 'raw' if !$format;
5617                 $devinfo->{$devname}->{devname} = $devname;
5618                 $devinfo->{$devname}->{virtdev} = $virtdev;
5619                 $devinfo->{$devname}->{format} = $format;
5620                 $devinfo->{$devname}->{storeid} = $storeid;
5621
5622                 # check permission on storage
5623                 my $pool = $opts->{pool}; # todo: do we need that?
5624                 if ($user ne 'root@pam') {
5625                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5626                 }
5627
5628                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5629             }
5630         }
5631
5632         foreach my $devname (keys %$devinfo) {
5633             die "found no device mapping information for device '$devname'\n"
5634                 if !$devinfo->{$devname}->{virtdev};
5635         }
5636
5637         my $cfg = PVE::Storage::config();
5638
5639         # create empty/temp config
5640         if ($oldconf) {
5641             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5642             foreach_drive($oldconf, sub {
5643                 my ($ds, $drive) = @_;
5644
5645                 return if drive_is_cdrom($drive);
5646
5647                 my $volid = $drive->{file};
5648
5649                 return if !$volid || $volid =~ m|^/|;
5650
5651                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5652                 return if !$path || !$owner || ($owner != $vmid);
5653
5654                 # Note: only delete disk we want to restore
5655                 # other volumes will become unused
5656                 if ($virtdev_hash->{$ds}) {
5657                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5658                     if (my $err = $@) {
5659                         warn $err;
5660                     }
5661                 }
5662             });
5663
5664             # delete vmstate files
5665             # since after the restore we have no snapshots anymore
5666             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5667                 my $snap = $oldconf->{snapshots}->{$snapname};
5668                 if ($snap->{vmstate}) {
5669                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5670                     if (my $err = $@) {
5671                         warn $err;
5672                     }
5673                 }
5674             }
5675         }
5676
5677         my $map = {};
5678         foreach my $virtdev (sort keys %$virtdev_hash) {
5679             my $d = $virtdev_hash->{$virtdev};
5680             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5681             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5682
5683             # test if requested format is supported
5684             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5685             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5686             $d->{format} = $defFormat if !$supported;
5687
5688             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5689                                                   $d->{format}, undef, $alloc_size);
5690             print STDERR "new volume ID is '$volid'\n";
5691             $d->{volid} = $volid;
5692             my $path = PVE::Storage::path($cfg, $volid);
5693
5694             PVE::Storage::activate_volumes($cfg,[$volid]);
5695
5696             my $write_zeros = 1;
5697             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5698                 $write_zeros = 0;
5699             }
5700
5701             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5702
5703             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5704             $map->{$virtdev} = $volid;
5705         }
5706
5707         $fh->seek(0, 0) || die "seek failed - $!\n";
5708
5709         my $outfd = new IO::File ($tmpfn, "w") ||
5710             die "unable to write config for VM $vmid\n";
5711
5712         my $cookie = { netcount => 0 };
5713         while (defined(my $line = <$fh>)) {
5714             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5715         }
5716
5717         $fh->close();
5718         $outfd->close();
5719     };
5720
5721     eval {
5722         # enable interrupts
5723         local $SIG{INT} =
5724             local $SIG{TERM} =
5725             local $SIG{QUIT} =
5726             local $SIG{HUP} =
5727             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5728         local $SIG{ALRM} = sub { die "got timeout\n"; };
5729
5730         $oldtimeout = alarm($timeout);
5731
5732         my $parser = sub {
5733             my $line = shift;
5734
5735             print "$line\n";
5736
5737             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5738                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5739                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5740             } elsif ($line =~ m/^CTIME: /) {
5741                 # we correctly received the vma config, so we can disable
5742                 # the timeout now for disk allocation (set to 10 minutes, so
5743                 # that we always timeout if something goes wrong)
5744                 alarm(600);
5745                 &$print_devmap();
5746                 print $fifofh "done\n";
5747                 my $tmp = $oldtimeout || 0;
5748                 $oldtimeout = undef;
5749                 alarm($tmp);
5750                 close($fifofh);
5751             }
5752         };
5753
5754         print "restore vma archive: $cmd\n";
5755         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5756     };
5757     my $err = $@;
5758
5759     alarm($oldtimeout) if $oldtimeout;
5760
5761     my $vollist = [];
5762     foreach my $devname (keys %$devinfo) {
5763         my $volid = $devinfo->{$devname}->{volid};
5764         push @$vollist, $volid if $volid;
5765     }
5766
5767     my $cfg = PVE::Storage::config();
5768     PVE::Storage::deactivate_volumes($cfg, $vollist);
5769
5770     unlink $mapfifo;
5771
5772     if ($err) {
5773         rmtree $tmpdir;
5774         unlink $tmpfn;
5775
5776         foreach my $devname (keys %$devinfo) {
5777             my $volid = $devinfo->{$devname}->{volid};
5778             next if !$volid;
5779             eval {
5780                 if ($volid =~ m|^/|) {
5781                     unlink $volid || die 'unlink failed\n';
5782                 } else {
5783                     PVE::Storage::vdisk_free($cfg, $volid);
5784                 }
5785                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5786             };
5787             print STDERR "unable to cleanup '$volid' - $@" if $@;
5788         }
5789         die $err;
5790     }
5791
5792     rmtree $tmpdir;
5793
5794     rename($tmpfn, $conffile) ||
5795         die "unable to commit configuration file '$conffile'\n";
5796
5797     PVE::Cluster::cfs_update(); # make sure we read new file
5798
5799     eval { rescan($vmid, 1); };
5800     warn $@ if $@;
5801 }
5802
5803 sub restore_tar_archive {
5804     my ($archive, $vmid, $user, $opts) = @_;
5805
5806     if ($archive ne '-') {
5807         my $firstfile = tar_archive_read_firstfile($archive);
5808         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5809             if $firstfile ne 'qemu-server.conf';
5810     }
5811
5812     my $storecfg = PVE::Storage::config();
5813
5814     # destroy existing data - keep empty config
5815     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5816     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5817
5818     my $tocmd = "/usr/lib/qemu-server/qmextract";
5819
5820     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5821     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5822     $tocmd .= ' --prealloc' if $opts->{prealloc};
5823     $tocmd .= ' --info' if $opts->{info};
5824
5825     # tar option "xf" does not autodetect compression when read from STDIN,
5826     # so we pipe to zcat
5827     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5828         PVE::Tools::shellquote("--to-command=$tocmd");
5829
5830     my $tmpdir = "/var/tmp/vzdumptmp$$";
5831     mkpath $tmpdir;
5832
5833     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5834     local $ENV{VZDUMP_VMID} = $vmid;
5835     local $ENV{VZDUMP_USER} = $user;
5836
5837     my $conffile = PVE::QemuConfig->config_file($vmid);
5838     my $tmpfn = "$conffile.$$.tmp";
5839
5840     # disable interrupts (always do cleanups)
5841     local $SIG{INT} =
5842         local $SIG{TERM} =
5843         local $SIG{QUIT} =
5844         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5845
5846     eval {
5847         # enable interrupts
5848         local $SIG{INT} =
5849             local $SIG{TERM} =
5850             local $SIG{QUIT} =
5851             local $SIG{HUP} =
5852             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5853
5854         if ($archive eq '-') {
5855             print "extracting archive from STDIN\n";
5856             run_command($cmd, input => "<&STDIN");
5857         } else {
5858             print "extracting archive '$archive'\n";
5859             run_command($cmd);
5860         }
5861
5862         return if $opts->{info};
5863
5864         # read new mapping
5865         my $map = {};
5866         my $statfile = "$tmpdir/qmrestore.stat";
5867         if (my $fd = IO::File->new($statfile, "r")) {
5868             while (defined (my $line = <$fd>)) {
5869                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5870                     $map->{$1} = $2 if $1;
5871                 } else {
5872                     print STDERR "unable to parse line in statfile - $line\n";
5873                 }
5874             }
5875             $fd->close();
5876         }
5877
5878         my $confsrc = "$tmpdir/qemu-server.conf";
5879
5880         my $srcfd = new IO::File($confsrc, "r") ||
5881             die "unable to open file '$confsrc'\n";
5882
5883         my $outfd = new IO::File ($tmpfn, "w") ||
5884             die "unable to write config for VM $vmid\n";
5885
5886         my $cookie = { netcount => 0 };
5887         while (defined (my $line = <$srcfd>)) {
5888             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5889         }
5890
5891         $srcfd->close();
5892         $outfd->close();
5893     };
5894     my $err = $@;
5895
5896     if ($err) {
5897
5898         unlink $tmpfn;
5899
5900         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5901
5902         die $err;
5903     }
5904
5905     rmtree $tmpdir;
5906
5907     rename $tmpfn, $conffile ||
5908         die "unable to commit configuration file '$conffile'\n";
5909
5910     PVE::Cluster::cfs_update(); # make sure we read new file
5911
5912     eval { rescan($vmid, 1); };
5913     warn $@ if $@;
5914 };
5915
5916 sub foreach_storage_used_by_vm {
5917     my ($conf, $func) = @_;
5918
5919     my $sidhash = {};
5920
5921     foreach_drive($conf, sub {
5922         my ($ds, $drive) = @_;
5923         return if drive_is_cdrom($drive);
5924
5925         my $volid = $drive->{file};
5926
5927         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5928         $sidhash->{$sid} = $sid if $sid;
5929     });
5930
5931     foreach my $sid (sort keys %$sidhash) {
5932         &$func($sid);
5933     }
5934 }
5935
5936 sub do_snapshots_with_qemu {
5937     my ($storecfg, $volid) = @_;
5938
5939     my $storage_name = PVE::Storage::parse_volume_id($volid);
5940
5941     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5942         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5943         return 1;
5944     }
5945
5946     if ($volid =~ m/\.(qcow2|qed)$/){
5947         return 1;
5948     }
5949
5950     return undef;
5951 }
5952
5953 sub qga_check_running {
5954     my ($vmid) = @_;
5955
5956     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5957     if ($@) {
5958         warn "Qemu Guest Agent is not running - $@";
5959         return 0;
5960     }
5961     return 1;
5962 }
5963
5964 sub template_create {
5965     my ($vmid, $conf, $disk) = @_;
5966
5967     my $storecfg = PVE::Storage::config();
5968
5969     foreach_drive($conf, sub {
5970         my ($ds, $drive) = @_;
5971
5972         return if drive_is_cdrom($drive);
5973         return if $disk && $ds ne $disk;
5974
5975         my $volid = $drive->{file};
5976         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5977
5978         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5979         $drive->{file} = $voliddst;
5980         $conf->{$ds} = print_drive($vmid, $drive);
5981         PVE::QemuConfig->write_config($vmid, $conf);
5982     });
5983 }
5984
5985 sub qemu_img_convert {
5986     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5987
5988     my $storecfg = PVE::Storage::config();
5989     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5990     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5991
5992     if ($src_storeid && $dst_storeid) {
5993
5994         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5995
5996         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5997         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5998
5999         my $src_format = qemu_img_format($src_scfg, $src_volname);
6000         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6001
6002         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6003         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6004
6005         my $cmd = [];
6006         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6007         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6008         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6009         if ($is_zero_initialized) {
6010             push @$cmd, "zeroinit:$dst_path";
6011         } else {
6012             push @$cmd, $dst_path;
6013         }
6014
6015         my $parser = sub {
6016             my $line = shift;
6017             if($line =~ m/\((\S+)\/100\%\)/){
6018                 my $percent = $1;
6019                 my $transferred = int($size * $percent / 100);
6020                 my $remaining = $size - $transferred;
6021
6022                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6023             }
6024
6025         };
6026
6027         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6028         my $err = $@;
6029         die "copy failed: $err" if $err;
6030     }
6031 }
6032
6033 sub qemu_img_format {
6034     my ($scfg, $volname) = @_;
6035
6036     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6037         return $1;
6038     } else {
6039         return "raw";
6040     }
6041 }
6042
6043 sub qemu_drive_mirror {
6044     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6045
6046     $jobs = {} if !$jobs;
6047
6048     my $qemu_target;
6049     my $format;
6050     $jobs->{"drive-$drive"} = {};
6051
6052     if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
6053         my $server = $1;
6054         my $port = $2;
6055         my $exportname = $3;
6056
6057         $format = "nbd";
6058         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
6059         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
6060         my $cmd = ['socat', '-T30', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=5"];
6061
6062         my $pid = fork();
6063         if (!defined($pid)) {
6064             die "forking socat tunnel failed\n";
6065         } elsif ($pid == 0) {
6066             exec(@$cmd);
6067             warn "exec failed: $!\n";
6068             POSIX::_exit(-1);
6069         }
6070         $jobs->{"drive-$drive"}->{pid} = $pid;
6071
6072         my $timeout = 0;
6073         while (!-S $unixsocket) {
6074             die "nbd connection helper timed out\n"
6075                 if $timeout++ > 5;
6076             sleep 1;
6077         }
6078     } else {
6079         my $storecfg = PVE::Storage::config();
6080         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6081
6082         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6083
6084         $format = qemu_img_format($dst_scfg, $dst_volname);
6085
6086         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6087
6088         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6089     }
6090
6091     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6092     $opts->{format} = $format if $format;
6093
6094     print "drive mirror is starting for drive-$drive\n";
6095
6096     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6097
6098     if (my $err = $@) {
6099         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6100         die "mirroring error: $err";
6101     }
6102
6103     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6104 }
6105
6106 sub qemu_drive_mirror_monitor {
6107     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6108
6109     eval {
6110         my $err_complete = 0;
6111
6112         while (1) {
6113             die "storage migration timed out\n" if $err_complete > 300;
6114
6115             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6116
6117             my $running_mirror_jobs = {};
6118             foreach my $stat (@$stats) {
6119                 next if $stat->{type} ne 'mirror';
6120                 $running_mirror_jobs->{$stat->{device}} = $stat;
6121             }
6122
6123             my $readycounter = 0;
6124
6125             foreach my $job (keys %$jobs) {
6126
6127                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6128                     print "$job : finished\n";
6129                     delete $jobs->{$job};
6130                     next;
6131                 }
6132
6133                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6134
6135                 my $busy = $running_mirror_jobs->{$job}->{busy};
6136                 my $ready = $running_mirror_jobs->{$job}->{ready};
6137                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6138                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6139                     my $remaining = $total - $transferred;
6140                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6141
6142                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6143                 }
6144
6145                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6146             }
6147
6148             last if scalar(keys %$jobs) == 0;
6149
6150             if ($readycounter == scalar(keys %$jobs)) {
6151                 print "all mirroring jobs are ready \n";
6152                 last if $skipcomplete; #do the complete later
6153
6154                 if ($vmiddst && $vmiddst != $vmid) {
6155                     my $agent_running = $qga && qga_check_running($vmid);
6156                     if ($agent_running) {
6157                         print "freeze filesystem\n";
6158                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6159                     } else {
6160                         print "suspend vm\n";
6161                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6162                     }
6163
6164                     # if we clone a disk for a new target vm, we don't switch the disk
6165                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6166
6167                     if ($agent_running) {
6168                         print "unfreeze filesystem\n";
6169                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6170                     } else {
6171                         print "resume vm\n";
6172                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6173                     }
6174
6175                     last;
6176                 } else {
6177
6178                     foreach my $job (keys %$jobs) {
6179                         # try to switch the disk if source and destination are on the same guest
6180                         print "$job: Completing block job...\n";
6181
6182                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6183                         if ($@ =~ m/cannot be completed/) {
6184                             print "$job: Block job cannot be completed, try again.\n";
6185                             $err_complete++;
6186                         }else {
6187                             print "$job: Completed successfully.\n";
6188                             $jobs->{$job}->{complete} = 1;
6189                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6190                         }
6191                     }
6192                 }
6193             }
6194             sleep 1;
6195         }
6196     };
6197     my $err = $@;
6198
6199     if ($err) {
6200         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6201         die "mirroring error: $err";
6202     }
6203
6204 }
6205
6206 sub qemu_blockjobs_cancel {
6207     my ($vmid, $jobs) = @_;
6208
6209     foreach my $job (keys %$jobs) {
6210         print "$job: Cancelling block job\n";
6211         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6212         $jobs->{$job}->{cancel} = 1;
6213     }
6214
6215     while (1) {
6216         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6217
6218         my $running_jobs = {};
6219         foreach my $stat (@$stats) {
6220             $running_jobs->{$stat->{device}} = $stat;
6221         }
6222
6223         foreach my $job (keys %$jobs) {
6224
6225             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6226                 print "$job: Done.\n";
6227                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6228                 delete $jobs->{$job};
6229             }
6230         }
6231
6232         last if scalar(keys %$jobs) == 0;
6233
6234         sleep 1;
6235     }
6236 }
6237
6238 sub qemu_blockjobs_finish_tunnel {
6239    my ($vmid, $job, $cpid) = @_;
6240
6241    return if !$cpid;
6242
6243    for (my $i = 1; $i < 20; $i++) {
6244         my $waitpid = waitpid($cpid, WNOHANG);
6245         last if (defined($waitpid) && ($waitpid == $cpid));
6246
6247         if ($i == 10) {
6248             kill(15, $cpid);
6249         } elsif ($i >= 15) {
6250             kill(9, $cpid);
6251         }
6252         sleep (1);
6253     }
6254     unlink "/run/qemu-server/$vmid.mirror-$job";
6255 }
6256
6257 sub clone_disk {
6258     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6259         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6260
6261     my $newvolid;
6262
6263     if (!$full) {
6264         print "create linked clone of drive $drivename ($drive->{file})\n";
6265         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6266         push @$newvollist, $newvolid;
6267     } else {
6268
6269         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6270         $storeid = $storage if $storage;
6271
6272         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6273         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6274
6275         print "create full clone of drive $drivename ($drive->{file})\n";
6276         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, undef, ($size/1024));
6277         push @$newvollist, $newvolid;
6278
6279         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6280
6281         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6282         if (!$running || $snapname) {
6283             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6284         } else {
6285
6286             my $kvmver = get_running_qemu_version ($vmid);
6287             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6288                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6289                     if $drive->{iothread};
6290             }
6291
6292             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6293         }
6294     }
6295
6296     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6297
6298     my $disk = $drive;
6299     $disk->{format} = undef;
6300     $disk->{file} = $newvolid;
6301     $disk->{size} = $size;
6302
6303     return $disk;
6304 }
6305
6306 # this only works if VM is running
6307 sub get_current_qemu_machine {
6308     my ($vmid) = @_;
6309
6310     my $cmd = { execute => 'query-machines', arguments => {} };
6311     my $res = vm_qmp_command($vmid, $cmd);
6312
6313     my ($current, $default);
6314     foreach my $e (@$res) {
6315         $default = $e->{name} if $e->{'is-default'};
6316         $current = $e->{name} if $e->{'is-current'};
6317     }
6318
6319     # fallback to the default machine if current is not supported by qemu
6320     return $current || $default || 'pc';
6321 }
6322
6323 sub get_running_qemu_version {
6324     my ($vmid) = @_;
6325     my $cmd = { execute => 'query-version', arguments => {} };
6326     my $res = vm_qmp_command($vmid, $cmd);
6327     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6328 }
6329
6330 sub qemu_machine_feature_enabled {
6331     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6332
6333     my $current_major;
6334     my $current_minor;
6335
6336     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6337
6338         $current_major = $3;
6339         $current_minor = $4;
6340
6341     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6342
6343         $current_major = $1;
6344         $current_minor = $2;
6345     }
6346
6347     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6348
6349
6350 }
6351
6352 sub qemu_machine_pxe {
6353     my ($vmid, $conf, $machine) = @_;
6354
6355     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6356
6357     foreach my $opt (keys %$conf) {
6358         next if $opt !~ m/^net(\d+)$/;
6359         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6360         next if !$net;
6361         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6362         return $machine.".pxe" if $romfile =~ m/pxe/;
6363         last;
6364     }
6365
6366     return $machine;
6367 }
6368
6369 sub qemu_use_old_bios_files {
6370     my ($machine_type) = @_;
6371
6372     return if !$machine_type;
6373
6374     my $use_old_bios_files = undef;
6375
6376     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6377         $machine_type = $1;
6378         $use_old_bios_files = 1;
6379     } else {
6380         my $kvmver = kvm_user_version();
6381         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6382         # load new efi bios files on migration. So this hack is required to allow
6383         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6384         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6385         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6386     }
6387
6388     return ($use_old_bios_files, $machine_type);
6389 }
6390
6391 sub create_efidisk {
6392     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6393
6394     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6395
6396     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6397     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6398     PVE::Storage::activate_volumes($storecfg, [$volid]);
6399
6400     my $path = PVE::Storage::path($storecfg, $volid);
6401     eval {
6402         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6403     };
6404     die "Copying EFI vars image failed: $@" if $@;
6405
6406     return ($volid, $vars_size);
6407 }
6408
6409 sub lspci {
6410
6411     my $devices = {};
6412
6413     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6414             my (undef, $id, $function) = @_;
6415             my $res = { id => $id, function => $function};
6416             push @{$devices->{$id}}, $res;
6417     });
6418
6419     # Entries should be sorted by functions.
6420     foreach my $id (keys %$devices) {
6421         my $dev = $devices->{$id};
6422         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6423     }
6424
6425     return $devices;
6426 }
6427
6428 sub vm_iothreads_list {
6429     my ($vmid) = @_;
6430
6431     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6432
6433     my $iothreads = {};
6434     foreach my $iothread (@$res) {
6435         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6436     }
6437
6438     return $iothreads;
6439 }
6440
6441 sub scsihw_infos {
6442     my ($conf, $drive) = @_;
6443
6444     my $maxdev = 0;
6445
6446     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6447         $maxdev = 7;
6448     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6449         $maxdev = 1;
6450     } else {
6451         $maxdev = 256;
6452     }
6453
6454     my $controller = int($drive->{index} / $maxdev);
6455     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6456
6457     return ($maxdev, $controller, $controller_prefix);
6458 }
6459
6460 sub add_hyperv_enlightenments {
6461     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6462
6463     return if $winversion < 6;
6464     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6465
6466     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6467
6468     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6469         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6470         push @$cpuFlags , 'hv_vapic';
6471         push @$cpuFlags , 'hv_time';
6472     } else {
6473         push @$cpuFlags , 'hv_spinlocks=0xffff';
6474     }
6475
6476     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6477         push @$cpuFlags , 'hv_reset';
6478         push @$cpuFlags , 'hv_vpindex';
6479         push @$cpuFlags , 'hv_runtime';
6480     }
6481
6482     if ($winversion >= 7) {
6483         push @$cpuFlags , 'hv_relaxed';
6484     }
6485 }
6486
6487 sub windows_version {
6488     my ($ostype) = @_;
6489
6490     return 0 if !$ostype;
6491
6492     my $winversion = 0;
6493
6494     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6495         $winversion = 5;
6496     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6497         $winversion = 6;
6498     } elsif ($ostype =~ m/^win(\d+)$/) {
6499         $winversion = $1;
6500     }
6501
6502     return $winversion;
6503 }
6504
6505 sub resolve_dst_disk_format {
6506         my ($storecfg, $storeid, $src_volname, $format) = @_;
6507         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6508
6509         if (!$format) {
6510             # if no target format is specified, use the source disk format as hint
6511             if ($src_volname) {
6512                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6513                 $format = qemu_img_format($scfg, $src_volname);
6514             } else {
6515                 return $defFormat;
6516             }
6517         }
6518
6519         # test if requested format is supported - else use default
6520         my $supported = grep { $_ eq $format } @$validFormats;
6521         $format = $defFormat if !$supported;
6522         return $format;
6523 }
6524
6525 sub resolve_first_disk {
6526     my $conf = shift;
6527     my @disks = PVE::QemuServer::valid_drive_names();
6528     my $firstdisk;
6529     foreach my $ds (reverse @disks) {
6530         next if !$conf->{$ds};
6531         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6532         next if PVE::QemuServer::drive_is_cdrom($disk);
6533         $firstdisk = $ds;
6534     }
6535     return $firstdisk;
6536 }
6537
6538 sub generate_smbios1_uuid {
6539     my ($uuid, $uuid_str);
6540     UUID::generate($uuid);
6541     UUID::unparse($uuid, $uuid_str);
6542     return "uuid=$uuid_str";
6543 }
6544
6545 # bash completion helper
6546
6547 sub complete_backup_archives {
6548     my ($cmdname, $pname, $cvalue) = @_;
6549
6550     my $cfg = PVE::Storage::config();
6551
6552     my $storeid;
6553
6554     if ($cvalue =~ m/^([^:]+):/) {
6555         $storeid = $1;
6556     }
6557
6558     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6559
6560     my $res = [];
6561     foreach my $id (keys %$data) {
6562         foreach my $item (@{$data->{$id}}) {
6563             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6564             push @$res, $item->{volid} if defined($item->{volid});
6565         }
6566     }
6567
6568     return $res;
6569 }
6570
6571 my $complete_vmid_full = sub {
6572     my ($running) = @_;
6573
6574     my $idlist = vmstatus();
6575
6576     my $res = [];
6577
6578     foreach my $id (keys %$idlist) {
6579         my $d = $idlist->{$id};
6580         if (defined($running)) {
6581             next if $d->{template};
6582             next if $running && $d->{status} ne 'running';
6583             next if !$running && $d->{status} eq 'running';
6584         }
6585         push @$res, $id;
6586
6587     }
6588     return $res;
6589 };
6590
6591 sub complete_vmid {
6592     return &$complete_vmid_full();
6593 }
6594
6595 sub complete_vmid_stopped {
6596     return &$complete_vmid_full(0);
6597 }
6598
6599 sub complete_vmid_running {
6600     return &$complete_vmid_full(1);
6601 }
6602
6603 sub complete_storage {
6604
6605     my $cfg = PVE::Storage::config();
6606     my $ids = $cfg->{ids};
6607
6608     my $res = [];
6609     foreach my $sid (keys %$ids) {
6610         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6611         next if !$ids->{$sid}->{content}->{images};
6612         push @$res, $sid;
6613     }
6614
6615     return $res;
6616 }
6617
6618 sub nbd_stop {
6619     my ($vmid) = @_;
6620
6621     vm_mon_cmd($vmid, 'nbd-server-stop');
6622 }
6623
6624 1;