PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42
  43 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  44
  45 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  46
  47 # Note about locking: we use flock on the config file protect
  48 # against concurent actions.
  49 # Aditionaly, we have a 'lock' setting in the config file. This
  50 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  51 # allowed when such lock is set. But you can ignore this kind of
  52 # lock with the --skiplock flag.
  53
  54 cfs_register_file('/qemu-server/',
  55                   \&parse_vm_config,
  56                   \&write_vm_config);
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  72     type => 'string',
  73     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  74     description => "The drive's backing file's data format.",
  75     optional => 1,
  76 });
  77
  78 #no warnings 'redefine';
  79
  80 sub cgroups_write {
  81    my ($controller, $vmid, $option, $value) = @_;
  82
  83    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  84    PVE::ProcFSTools::write_proc_entry($path, $value);
  85
  86 }
  87
  88 my $nodename = PVE::INotify::nodename();
  89
  90 mkdir "/etc/pve/nodes/$nodename";
  91 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  92 mkdir $confdir;
  93
  94 my $var_run_tmpdir = "/var/run/qemu-server";
  95 mkdir $var_run_tmpdir;
  96
  97 my $lock_dir = "/var/lock/qemu-server";
  98 mkdir $lock_dir;
  99
 100 my $pcisysfs = "/sys/bus/pci";
 101
 102 my $cpu_vendor_list = {
 103     # Intel CPUs
 104     486 => 'GenuineIntel',
 105     pentium => 'GenuineIntel',
 106     pentium2  => 'GenuineIntel',
 107     pentium3  => 'GenuineIntel',
 108     coreduo => 'GenuineIntel',
 109     core2duo => 'GenuineIntel',
 110     Conroe  => 'GenuineIntel',
 111     Penryn  => 'GenuineIntel',
 112     Nehalem  => 'GenuineIntel',
 113     'Nehalem-IBRS'  => 'GenuineIntel',
 114     Westmere => 'GenuineIntel',
 115     'Westmere-IBRS' => 'GenuineIntel',
 116     SandyBridge => 'GenuineIntel',
 117     'SandyBridge-IBRS' => 'GenuineIntel',
 118     IvyBridge => 'GenuineIntel',
 119     'IvyBridge-IBRS' => 'GenuineIntel',
 120     Haswell => 'GenuineIntel',
 121     'Haswell-IBRS' => 'GenuineIntel',
 122     'Haswell-noTSX' => 'GenuineIntel',
 123     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 124     Broadwell => 'GenuineIntel',
 125     'Broadwell-IBRS' => 'GenuineIntel',
 126     'Broadwell-noTSX' => 'GenuineIntel',
 127     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 128     'Skylake-Client' => 'GenuineIntel',
 129     'Skylake-Client-IBRS' => 'GenuineIntel',
 130     'Skylake-Server' => 'GenuineIntel',
 131     'Skylake-Server-IBRS' => 'GenuineIntel',
 132
 133     # AMD CPUs
 134     athlon => 'AuthenticAMD',
 135     phenom  => 'AuthenticAMD',
 136     Opteron_G1  => 'AuthenticAMD',
 137     Opteron_G2  => 'AuthenticAMD',
 138     Opteron_G3  => 'AuthenticAMD',
 139     Opteron_G4  => 'AuthenticAMD',
 140     Opteron_G5  => 'AuthenticAMD',
 141
 142     # generic types, use vendor from host node
 143     host => 'default',
 144     kvm32 => 'default',
 145     kvm64 => 'default',
 146     qemu32 => 'default',
 147     qemu64 => 'default',
 148     max => 'default',
 149 };
 150
 151 my $cpu_flag = qr/[+-](pcid|spec-ctrl)/;
 152
 153 my $cpu_fmt = {
 154     cputype => {
 155         description => "Emulated CPU type.",
 156         type => 'string',
 157         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 158         default => 'kvm64',
 159         default_key => 1,
 160     },
 161     hidden => {
 162         description => "Do not identify as a KVM virtual machine.",
 163         type => 'boolean',
 164         optional => 1,
 165         default => 0
 166     },
 167     flags => {
 168         description => "List of additional CPU flags separated by ';'."
 169                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 170                      . " Currently supported flags: 'pcid', 'spec-ctrl'.",
 171         format_description => '+FLAG[;-FLAG...]',
 172         type => 'string',
 173         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 174         optional => 1,
 175     },
 176 };
 177
 178 my $watchdog_fmt = {
 179     model => {
 180         default_key => 1,
 181         type => 'string',
 182         enum => [qw(i6300esb ib700)],
 183         description => "Watchdog type to emulate.",
 184         default => 'i6300esb',
 185         optional => 1,
 186     },
 187     action => {
 188         type => 'string',
 189         enum => [qw(reset shutdown poweroff pause debug none)],
 190         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 191         optional => 1,
 192     },
 193 };
 194 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 195
 196 my $confdesc = {
 197     onboot => {
 198         optional => 1,
 199         type => 'boolean',
 200         description => "Specifies whether a VM will be started during system bootup.",
 201         default => 0,
 202     },
 203     autostart => {
 204         optional => 1,
 205         type => 'boolean',
 206         description => "Automatic restart after crash (currently ignored).",
 207         default => 0,
 208     },
 209     hotplug => {
 210         optional => 1,
 211         type => 'string', format => 'pve-hotplug-features',
 212         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 213         default => 'network,disk,usb',
 214     },
 215     reboot => {
 216         optional => 1,
 217         type => 'boolean',
 218         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 219         default => 1,
 220     },
 221     lock => {
 222         optional => 1,
 223         type => 'string',
 224         description => "Lock/unlock the VM.",
 225         enum => [qw(migrate backup snapshot rollback)],
 226     },
 227     cpulimit => {
 228         optional => 1,
 229         type => 'number',
 230         description => "Limit of CPU usage.",
 231         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 232         minimum => 0,
 233         maximum => 128,
 234         default => 0,
 235     },
 236     cpuunits => {
 237         optional => 1,
 238         type => 'integer',
 239         description => "CPU weight for a VM.",
 240         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 241         minimum => 2,
 242         maximum => 262144,
 243         default => 1024,
 244     },
 245     memory => {
 246         optional => 1,
 247         type => 'integer',
 248         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 249         minimum => 16,
 250         default => 512,
 251     },
 252     balloon => {
 253         optional => 1,
 254         type => 'integer',
 255         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 256         minimum => 0,
 257     },
 258     shares => {
 259         optional => 1,
 260         type => 'integer',
 261         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 262         minimum => 0,
 263         maximum => 50000,
 264         default => 1000,
 265     },
 266     keyboard => {
 267         optional => 1,
 268         type => 'string',
 269         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.".
 270                        "It should not be necessary to set it.",
 271         enum => PVE::Tools::kvmkeymaplist(),
 272         default => undef,
 273     },
 274     name => {
 275         optional => 1,
 276         type => 'string', format => 'dns-name',
 277         description => "Set a name for the VM. Only used on the configuration web interface.",
 278     },
 279     scsihw => {
 280         optional => 1,
 281         type => 'string',
 282         description => "SCSI controller model",
 283         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 284         default => 'lsi',
 285     },
 286     description => {
 287         optional => 1,
 288         type => 'string',
 289         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 290     },
 291     ostype => {
 292         optional => 1,
 293         type => 'string',
 294         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 295         description => "Specify guest operating system.",
 296         verbose_description => <<EODESC,
 297 Specify guest operating system. This is used to enable special
 298 optimization/features for specific operating systems:
 299
 300 [horizontal]
 301 other;; unspecified OS
 302 wxp;; Microsoft Windows XP
 303 w2k;; Microsoft Windows 2000
 304 w2k3;; Microsoft Windows 2003
 305 w2k8;; Microsoft Windows 2008
 306 wvista;; Microsoft Windows Vista
 307 win7;; Microsoft Windows 7
 308 win8;; Microsoft Windows 8/2012/2012r2
 309 win10;; Microsoft Windows 10/2016
 310 l24;; Linux 2.4 Kernel
 311 l26;; Linux 2.6/3.X Kernel
 312 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 313 EODESC
 314     },
 315     boot => {
 316         optional => 1,
 317         type => 'string',
 318         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 319         pattern => '[acdn]{1,4}',
 320         default => 'cdn',
 321     },
 322     bootdisk => {
 323         optional => 1,
 324         type => 'string', format => 'pve-qm-bootdisk',
 325         description => "Enable booting from specified disk.",
 326         pattern => '(ide|sata|scsi|virtio)\d+',
 327     },
 328     smp => {
 329         optional => 1,
 330         type => 'integer',
 331         description => "The number of CPUs. Please use option -sockets instead.",
 332         minimum => 1,
 333         default => 1,
 334     },
 335     sockets => {
 336         optional => 1,
 337         type => 'integer',
 338         description => "The number of CPU sockets.",
 339         minimum => 1,
 340         default => 1,
 341     },
 342     cores => {
 343         optional => 1,
 344         type => 'integer',
 345         description => "The number of cores per socket.",
 346         minimum => 1,
 347         default => 1,
 348     },
 349     numa => {
 350         optional => 1,
 351         type => 'boolean',
 352         description => "Enable/disable NUMA.",
 353         default => 0,
 354     },
 355     hugepages => {
 356         optional => 1,
 357         type => 'string',
 358         description => "Enable/disable hugepages memory.",
 359         enum => [qw(any 2 1024)],
 360     },
 361     vcpus => {
 362         optional => 1,
 363         type => 'integer',
 364         description => "Number of hotplugged vcpus.",
 365         minimum => 1,
 366         default => 0,
 367     },
 368     acpi => {
 369         optional => 1,
 370         type => 'boolean',
 371         description => "Enable/disable ACPI.",
 372         default => 1,
 373     },
 374     agent => {
 375         optional => 1,
 376         type => 'boolean',
 377         description => "Enable/disable Qemu GuestAgent.",
 378         default => 0,
 379     },
 380     kvm => {
 381         optional => 1,
 382         type => 'boolean',
 383         description => "Enable/disable KVM hardware virtualization.",
 384         default => 1,
 385     },
 386     tdf => {
 387         optional => 1,
 388         type => 'boolean',
 389         description => "Enable/disable time drift fix.",
 390         default => 0,
 391     },
 392     localtime => {
 393         optional => 1,
 394         type => 'boolean',
 395         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 396     },
 397     freeze => {
 398         optional => 1,
 399         type => 'boolean',
 400         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 401     },
 402     vga => {
 403         optional => 1,
 404         type => 'string',
 405         description => "Select the VGA type.",
 406         verbose_description => "Select the VGA type. If you want to use high resolution" .
 407             " modes (>= 1280x1024x16) then you should use the options " .
 408             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 409             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 410             "display sever. For win* OS you can select how many independent " .
 411             "displays you want, Linux guests can add displays them self. " .
 412             "You can also run without any graphic card, using a serial device" .
 413             " as terminal.",
 414         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 415     },
 416     watchdog => {
 417         optional => 1,
 418         type => 'string', format => 'pve-qm-watchdog',
 419         description => "Create a virtual hardware watchdog device.",
 420         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 421             " (by a guest action), the watchdog must be periodically polled " .
 422             "by an agent inside the guest or else the watchdog will reset " .
 423             "the guest (or execute the respective action specified)",
 424     },
 425     startdate => {
 426         optional => 1,
 427         type => 'string',
 428         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 429         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 430         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 431         default => 'now',
 432     },
 433     startup =>  get_standard_option('pve-startup-order'),
 434     template => {
 435         optional => 1,
 436         type => 'boolean',
 437         description => "Enable/disable Template.",
 438         default => 0,
 439     },
 440     args => {
 441         optional => 1,
 442         type => 'string',
 443         description => "Arbitrary arguments passed to kvm.",
 444         verbose_description => <<EODESCR,
 445 Arbitrary arguments passed to kvm, for example:
 446
 447 args: -no-reboot -no-hpet
 448
 449 NOTE: this option is for experts only.
 450 EODESCR
 451     },
 452     tablet => {
 453         optional => 1,
 454         type => 'boolean',
 455         default => 1,
 456         description => "Enable/disable the USB tablet device.",
 457         verbose_description => "Enable/disable the USB tablet device. This device is " .
 458             "usually needed to allow absolute mouse positioning with VNC. " .
 459             "Else the mouse runs out of sync with normal VNC clients. " .
 460             "If you're running lots of console-only guests on one host, " .
 461             "you may consider disabling this to save some context switches. " .
 462             "This is turned off by default if you use spice (-vga=qxl).",
 463     },
 464     migrate_speed => {
 465         optional => 1,
 466         type => 'integer',
 467         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 468         minimum => 0,
 469         default => 0,
 470     },
 471     migrate_downtime => {
 472         optional => 1,
 473         type => 'number',
 474         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 475         minimum => 0,
 476         default => 0.1,
 477     },
 478     cdrom => {
 479         optional => 1,
 480         type => 'string', format => 'pve-qm-ide',
 481         typetext => '<volume>',
 482         description => "This is an alias for option -ide2",
 483     },
 484     cpu => {
 485         optional => 1,
 486         description => "Emulated CPU type.",
 487         type => 'string',
 488         format => $cpu_fmt,
 489     },
 490     parent => get_standard_option('pve-snapshot-name', {
 491         optional => 1,
 492         description => "Parent snapshot name. This is used internally, and should not be modified.",
 493     }),
 494     snaptime => {
 495         optional => 1,
 496         description => "Timestamp for snapshots.",
 497         type => 'integer',
 498         minimum => 0,
 499     },
 500     vmstate => {
 501         optional => 1,
 502         type => 'string', format => 'pve-volume-id',
 503         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 504     },
 505     vmstatestorage => get_standard_option('pve-storage-id', {
 506         description => "Default storage for VM state volumes/files.",
 507         optional => 1,
 508     }),
 509     machine => {
 510         description => "Specific the Qemu machine type.",
 511         type => 'string',
 512         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 513         maxLength => 40,
 514         optional => 1,
 515     },
 516     smbios1 => {
 517         description => "Specify SMBIOS type 1 fields.",
 518         type => 'string', format => 'pve-qm-smbios1',
 519         maxLength => 256,
 520         optional => 1,
 521     },
 522     protection => {
 523         optional => 1,
 524         type => 'boolean',
 525         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 526         default => 0,
 527     },
 528     bios => {
 529         optional => 1,
 530         type => 'string',
 531         enum => [ qw(seabios ovmf) ],
 532         description => "Select BIOS implementation.",
 533         default => 'seabios',
 534     },
 535 };
 536
 537 # what about other qemu settings ?
 538 #cpu => 'string',
 539 #machine => 'string',
 540 #fda => 'file',
 541 #fdb => 'file',
 542 #mtdblock => 'file',
 543 #sd => 'file',
 544 #pflash => 'file',
 545 #snapshot => 'bool',
 546 #bootp => 'file',
 547 ##tftp => 'dir',
 548 ##smb => 'dir',
 549 #kernel => 'file',
 550 #append => 'string',
 551 #initrd => 'file',
 552 ##soundhw => 'string',
 553
 554 while (my ($k, $v) = each %$confdesc) {
 555     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 556 }
 557
 558 my $MAX_IDE_DISKS = 4;
 559 my $MAX_SCSI_DISKS = 14;
 560 my $MAX_VIRTIO_DISKS = 16;
 561 my $MAX_SATA_DISKS = 6;
 562 my $MAX_USB_DEVICES = 5;
 563 my $MAX_NETS = 32;
 564 my $MAX_UNUSED_DISKS = 8;
 565 my $MAX_HOSTPCI_DEVICES = 4;
 566 my $MAX_SERIAL_PORTS = 4;
 567 my $MAX_PARALLEL_PORTS = 3;
 568 my $MAX_NUMA = 8;
 569
 570 my $numa_fmt = {
 571     cpus => {
 572         type => "string",
 573         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 574         description => "CPUs accessing this NUMA node.",
 575         format_description => "id[-id];...",
 576     },
 577     memory => {
 578         type => "number",
 579         description => "Amount of memory this NUMA node provides.",
 580         optional => 1,
 581     },
 582     hostnodes => {
 583         type => "string",
 584         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 585         description => "Host NUMA nodes to use.",
 586         format_description => "id[-id];...",
 587         optional => 1,
 588     },
 589     policy => {
 590         type => 'string',
 591         enum => [qw(preferred bind interleave)],
 592         description => "NUMA allocation policy.",
 593         optional => 1,
 594     },
 595 };
 596 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 597 my $numadesc = {
 598     optional => 1,
 599     type => 'string', format => $numa_fmt,
 600     description => "NUMA topology.",
 601 };
 602 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 603
 604 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 605     $confdesc->{"numa$i"} = $numadesc;
 606 }
 607
 608 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 609                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 610                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 611 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 612
 613 my $net_fmt_bridge_descr = <<__EOD__;
 614 Bridge to attach the network device to. The Proxmox VE standard bridge
 615 is called 'vmbr0'.
 616
 617 If you do not specify a bridge, we create a kvm user (NATed) network
 618 device, which provides DHCP and DNS services. The following addresses
 619 are used:
 620
 621  10.0.2.2   Gateway
 622  10.0.2.3   DNS Server
 623  10.0.2.4   SMB Server
 624
 625 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 626 __EOD__
 627
 628 my $net_fmt = {
 629     macaddr => {
 630         type => 'string',
 631         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 632         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 633         format_description => "XX:XX:XX:XX:XX:XX",
 634         optional => 1,
 635     },
 636     model => {
 637         type => 'string',
 638         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 639         enum => $nic_model_list,
 640         default_key => 1,
 641     },
 642     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 643     bridge => {
 644         type => 'string',
 645         description => $net_fmt_bridge_descr,
 646         format_description => 'bridge',
 647         optional => 1,
 648     },
 649     queues => {
 650         type => 'integer',
 651         minimum => 0, maximum => 16,
 652         description => 'Number of packet queues to be used on the device.',
 653         optional => 1,
 654     },
 655     rate => {
 656         type => 'number',
 657         minimum => 0,
 658         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 659         optional => 1,
 660     },
 661     tag => {
 662         type => 'integer',
 663         minimum => 1, maximum => 4094,
 664         description => 'VLAN tag to apply to packets on this interface.',
 665         optional => 1,
 666     },
 667     trunks => {
 668         type => 'string',
 669         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 670         description => 'VLAN trunks to pass through this interface.',
 671         format_description => 'vlanid[;vlanid...]',
 672         optional => 1,
 673     },
 674     firewall => {
 675         type => 'boolean',
 676         description => 'Whether this interface should be protected by the firewall.',
 677         optional => 1,
 678     },
 679     link_down => {
 680         type => 'boolean',
 681         description => 'Whether this interface should be disconnected (like pulling the plug).',
 682         optional => 1,
 683     },
 684 };
 685
 686 my $netdesc = {
 687     optional => 1,
 688     type => 'string', format => $net_fmt,
 689     description => "Specify network devices.",
 690 };
 691
 692 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 693
 694 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 695     $confdesc->{"net$i"} = $netdesc;
 696 }
 697
 698 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 699 sub verify_volume_id_or_qm_path {
 700     my ($volid, $noerr) = @_;
 701
 702     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 703         return $volid;
 704     }
 705
 706     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 707     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 708     if ($@) {
 709         return undef if $noerr;
 710         die $@;
 711     }
 712     return $volid;
 713 }
 714
 715 my $drivename_hash;
 716
 717 my %drivedesc_base = (
 718     volume => { alias => 'file' },
 719     file => {
 720         type => 'string',
 721         format => 'pve-volume-id-or-qm-path',
 722         default_key => 1,
 723         format_description => 'volume',
 724         description => "The drive's backing volume.",
 725     },
 726     media => {
 727         type => 'string',
 728         enum => [qw(cdrom disk)],
 729         description => "The drive's media type.",
 730         default => 'disk',
 731         optional => 1
 732     },
 733     cyls => {
 734         type => 'integer',
 735         description => "Force the drive's physical geometry to have a specific cylinder count.",
 736         optional => 1
 737     },
 738     heads => {
 739         type => 'integer',
 740         description => "Force the drive's physical geometry to have a specific head count.",
 741         optional => 1
 742     },
 743     secs => {
 744         type => 'integer',
 745         description => "Force the drive's physical geometry to have a specific sector count.",
 746         optional => 1
 747     },
 748     trans => {
 749         type => 'string',
 750         enum => [qw(none lba auto)],
 751         description => "Force disk geometry bios translation mode.",
 752         optional => 1,
 753     },
 754     snapshot => {
 755         type => 'boolean',
 756         description => "Controls qemu's snapshot mode feature."
 757             . " If activated, changes made to the disk are temporary and will"
 758             . " be discarded when the VM is shutdown.",
 759         optional => 1,
 760     },
 761     cache => {
 762         type => 'string',
 763         enum => [qw(none writethrough writeback unsafe directsync)],
 764         description => "The drive's cache mode",
 765         optional => 1,
 766     },
 767     format => get_standard_option('pve-qm-image-format'),
 768     size => {
 769         type => 'string',
 770         format => 'disk-size',
 771         format_description => 'DiskSize',
 772         description => "Disk size. This is purely informational and has no effect.",
 773         optional => 1,
 774     },
 775     backup => {
 776         type => 'boolean',
 777         description => "Whether the drive should be included when making backups.",
 778         optional => 1,
 779     },
 780     replicate => {
 781         type => 'boolean',
 782         description => 'Whether the drive should considered for replication jobs.',
 783         optional => 1,
 784         default => 1,
 785     },
 786     rerror => {
 787         type => 'string',
 788         enum => [qw(ignore report stop)],
 789         description => 'Read error action.',
 790         optional => 1,
 791     },
 792     werror => {
 793         type => 'string',
 794         enum => [qw(enospc ignore report stop)],
 795         description => 'Write error action.',
 796         optional => 1,
 797     },
 798     aio => {
 799         type => 'string',
 800         enum => [qw(native threads)],
 801         description => 'AIO type to use.',
 802         optional => 1,
 803     },
 804     discard => {
 805         type => 'string',
 806         enum => [qw(ignore on)],
 807         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 808         optional => 1,
 809     },
 810     detect_zeroes => {
 811         type => 'boolean',
 812         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 813         optional => 1,
 814     },
 815     serial => {
 816         type => 'string',
 817         format => 'urlencoded',
 818         format_description => 'serial',
 819         maxLength => 20*3, # *3 since it's %xx url enoded
 820         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 821         optional => 1,
 822     }
 823 );
 824
 825 my %iothread_fmt = ( iothread => {
 826         type => 'boolean',
 827         description => "Whether to use iothreads for this drive",
 828         optional => 1,
 829 });
 830
 831 my %model_fmt = (
 832     model => {
 833         type => 'string',
 834         format => 'urlencoded',
 835         format_description => 'model',
 836         maxLength => 40*3, # *3 since it's %xx url enoded
 837         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 838         optional => 1,
 839     },
 840 );
 841
 842 my %queues_fmt = (
 843     queues => {
 844         type => 'integer',
 845         description => "Number of queues.",
 846         minimum => 2,
 847         optional => 1
 848     }
 849 );
 850
 851 my %scsiblock_fmt = (
 852     scsiblock => {
 853         type => 'boolean',
 854         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 855         optional => 1,
 856         default => 0,
 857     },
 858 );
 859
 860 my $add_throttle_desc = sub {
 861     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 862     my $d = {
 863         type => $type,
 864         format_description => $unit,
 865         description => "Maximum $what in $longunit.",
 866         optional => 1,
 867     };
 868     $d->{minimum} = $minimum if defined($minimum);
 869     $drivedesc_base{$key} = $d;
 870 };
 871 # throughput: (leaky bucket)
 872 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 873 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 874 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 875 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 876 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 877 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 878 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 879 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 880 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 881
 882 # pools: (pool of IO before throttling starts taking effect)
 883 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 884 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 885 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 886 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 887 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 888 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 889
 890 # burst lengths
 891 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 892 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 893 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 894 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 895 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 896 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 897
 898 # legacy support
 899 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
 900 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
 901 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
 902 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
 903
 904 my $ide_fmt = {
 905     %drivedesc_base,
 906     %model_fmt,
 907 };
 908 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 909
 910 my $idedesc = {
 911     optional => 1,
 912     type => 'string', format => $ide_fmt,
 913     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 914 };
 915 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 916
 917 my $scsi_fmt = {
 918     %drivedesc_base,
 919     %iothread_fmt,
 920     %queues_fmt,
 921     %scsiblock_fmt,
 922 };
 923 my $scsidesc = {
 924     optional => 1,
 925     type => 'string', format => $scsi_fmt,
 926     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 927 };
 928 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 929
 930 my $sata_fmt = {
 931     %drivedesc_base,
 932 };
 933 my $satadesc = {
 934     optional => 1,
 935     type => 'string', format => $sata_fmt,
 936     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 937 };
 938 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 939
 940 my $virtio_fmt = {
 941     %drivedesc_base,
 942     %iothread_fmt,
 943 };
 944 my $virtiodesc = {
 945     optional => 1,
 946     type => 'string', format => $virtio_fmt,
 947     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 948 };
 949 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 950
 951 my $alldrive_fmt = {
 952     %drivedesc_base,
 953     %iothread_fmt,
 954     %model_fmt,
 955     %queues_fmt,
 956     %scsiblock_fmt,
 957 };
 958
 959 my $efidisk_fmt = {
 960     volume => { alias => 'file' },
 961     file => {
 962         type => 'string',
 963         format => 'pve-volume-id-or-qm-path',
 964         default_key => 1,
 965         format_description => 'volume',
 966         description => "The drive's backing volume.",
 967     },
 968     format => get_standard_option('pve-qm-image-format'),
 969     size => {
 970         type => 'string',
 971         format => 'disk-size',
 972         format_description => 'DiskSize',
 973         description => "Disk size. This is purely informational and has no effect.",
 974         optional => 1,
 975     },
 976 };
 977
 978 my $efidisk_desc = {
 979     optional => 1,
 980     type => 'string', format => $efidisk_fmt,
 981     description => "Configure a Disk for storing EFI vars",
 982 };
 983
 984 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 985
 986 my $usb_fmt = {
 987     host => {
 988         default_key => 1,
 989         type => 'string', format => 'pve-qm-usb-device',
 990         format_description => 'HOSTUSBDEVICE|spice',
 991         description => <<EODESCR,
 992 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 993
 994  'bus-port(.port)*' (decimal numbers) or
 995  'vendor_id:product_id' (hexadeciaml numbers) or
 996  'spice'
 997
 998 You can use the 'lsusb -t' command to list existing usb devices.
 999
1000 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1001
1002 The value 'spice' can be used to add a usb redirection devices for spice.
1003 EODESCR
1004     },
1005     usb3 => {
1006         optional => 1,
1007         type => 'boolean',
1008         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1009         default => 0,
1010     },
1011 };
1012
1013 my $usbdesc = {
1014     optional => 1,
1015     type => 'string', format => $usb_fmt,
1016     description => "Configure an USB device (n is 0 to 4).",
1017 };
1018 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1019
1020 # NOTE: the match-groups of this regex are used in parse_hostpci
1021 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1022 my $hostpci_fmt = {
1023     host => {
1024         default_key => 1,
1025         type => 'string',
1026         pattern => qr/$PCIRE(;$PCIRE)*/,
1027         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1028         description => <<EODESCR,
1029 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1030 of PCI virtual functions of the host. HOSTPCIID syntax is:
1031
1032 'bus:dev.func' (hexadecimal numbers)
1033
1034 You can us the 'lspci' command to list existing PCI devices.
1035 EODESCR
1036     },
1037     rombar => {
1038         type => 'boolean',
1039         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1040         optional => 1,
1041         default => 1,
1042     },
1043     romfile => {
1044         type => 'string',
1045         pattern => '[^,;]+',
1046         format_description => 'string',
1047         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1048         optional => 1,
1049     },
1050     pcie => {
1051         type => 'boolean',
1052         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1053         optional => 1,
1054         default => 0,
1055     },
1056     'x-vga' => {
1057         type => 'boolean',
1058         description =>  "Enable vfio-vga device support.",
1059         optional => 1,
1060         default => 0,
1061     },
1062 };
1063 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1064
1065 my $hostpcidesc = {
1066         optional => 1,
1067         type => 'string', format => 'pve-qm-hostpci',
1068         description => "Map host PCI devices into guest.",
1069         verbose_description =>  <<EODESCR,
1070 Map host PCI devices into guest.
1071
1072 NOTE: This option allows direct access to host hardware. So it is no longer
1073 possible to migrate such machines - use with special care.
1074
1075 CAUTION: Experimental! User reported problems with this option.
1076 EODESCR
1077 };
1078 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1079
1080 my $serialdesc = {
1081         optional => 1,
1082         type => 'string',
1083         pattern => '(/dev/.+|socket)',
1084         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1085         verbose_description =>  <<EODESCR,
1086 Create a serial device inside the VM (n is 0 to 3), and pass through a
1087 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1088 host side (use 'qm terminal' to open a terminal connection).
1089
1090 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1091
1092 CAUTION: Experimental! User reported problems with this option.
1093 EODESCR
1094 };
1095
1096 my $paralleldesc= {
1097         optional => 1,
1098         type => 'string',
1099         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1100         description =>  "Map host parallel devices (n is 0 to 2).",
1101         verbose_description =>  <<EODESCR,
1102 Map host parallel devices (n is 0 to 2).
1103
1104 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1105
1106 CAUTION: Experimental! User reported problems with this option.
1107 EODESCR
1108 };
1109
1110 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1111     $confdesc->{"parallel$i"} = $paralleldesc;
1112 }
1113
1114 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1115     $confdesc->{"serial$i"} = $serialdesc;
1116 }
1117
1118 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1119     $confdesc->{"hostpci$i"} = $hostpcidesc;
1120 }
1121
1122 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1123     $drivename_hash->{"ide$i"} = 1;
1124     $confdesc->{"ide$i"} = $idedesc;
1125 }
1126
1127 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1128     $drivename_hash->{"sata$i"} = 1;
1129     $confdesc->{"sata$i"} = $satadesc;
1130 }
1131
1132 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1133     $drivename_hash->{"scsi$i"} = 1;
1134     $confdesc->{"scsi$i"} = $scsidesc ;
1135 }
1136
1137 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1138     $drivename_hash->{"virtio$i"} = 1;
1139     $confdesc->{"virtio$i"} = $virtiodesc;
1140 }
1141
1142 $drivename_hash->{efidisk0} = 1;
1143 $confdesc->{efidisk0} = $efidisk_desc;
1144
1145 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1146     $confdesc->{"usb$i"} = $usbdesc;
1147 }
1148
1149 my $unuseddesc = {
1150     optional => 1,
1151     type => 'string', format => 'pve-volume-id',
1152     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1153 };
1154
1155 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1156     $confdesc->{"unused$i"} = $unuseddesc;
1157 }
1158
1159 my $kvm_api_version = 0;
1160
1161 sub kvm_version {
1162
1163     return $kvm_api_version if $kvm_api_version;
1164
1165     my $fh = IO::File->new("</dev/kvm") ||
1166         return 0;
1167
1168     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1169         $kvm_api_version = $v;
1170     }
1171
1172     $fh->close();
1173
1174     return  $kvm_api_version;
1175 }
1176
1177 my $kvm_user_version;
1178
1179 sub kvm_user_version {
1180
1181     return $kvm_user_version if $kvm_user_version;
1182
1183     $kvm_user_version = 'unknown';
1184
1185     my $code = sub {
1186         my $line = shift;
1187         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1188             $kvm_user_version = $2;
1189         }
1190     };
1191
1192     eval { run_command("kvm -version", outfunc => $code); };
1193     warn $@ if $@;
1194
1195     return $kvm_user_version;
1196
1197 }
1198
1199 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1200
1201 sub valid_drive_names {
1202     # order is important - used to autoselect boot disk
1203     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1204             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1205             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1206             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1207             'efidisk0');
1208 }
1209
1210 sub is_valid_drivename {
1211     my $dev = shift;
1212
1213     return defined($drivename_hash->{$dev});
1214 }
1215
1216 sub option_exists {
1217     my $key = shift;
1218     return defined($confdesc->{$key});
1219 }
1220
1221 sub nic_models {
1222     return $nic_model_list;
1223 }
1224
1225 sub os_list_description {
1226
1227     return {
1228         other => 'Other',
1229         wxp => 'Windows XP',
1230         w2k => 'Windows 2000',
1231         w2k3 =>, 'Windows 2003',
1232         w2k8 => 'Windows 2008',
1233         wvista => 'Windows Vista',
1234         win7 => 'Windows 7',
1235         win8 => 'Windows 8/2012',
1236         win10 => 'Windows 10/2016',
1237         l24 => 'Linux 2.4',
1238         l26 => 'Linux 2.6',
1239     };
1240 }
1241
1242 my $cdrom_path;
1243
1244 sub get_cdrom_path {
1245
1246     return  $cdrom_path if $cdrom_path;
1247
1248     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1249     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1250     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1251 }
1252
1253 sub get_iso_path {
1254     my ($storecfg, $vmid, $cdrom) = @_;
1255
1256     if ($cdrom eq 'cdrom') {
1257         return get_cdrom_path();
1258     } elsif ($cdrom eq 'none') {
1259         return '';
1260     } elsif ($cdrom =~ m|^/|) {
1261         return $cdrom;
1262     } else {
1263         return PVE::Storage::path($storecfg, $cdrom);
1264     }
1265 }
1266
1267 # try to convert old style file names to volume IDs
1268 sub filename_to_volume_id {
1269     my ($vmid, $file, $media) = @_;
1270
1271     if (!($file eq 'none' || $file eq 'cdrom' ||
1272           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1273
1274         return undef if $file =~ m|/|;
1275
1276         if ($media && $media eq 'cdrom') {
1277             $file = "local:iso/$file";
1278         } else {
1279             $file = "local:$vmid/$file";
1280         }
1281     }
1282
1283     return $file;
1284 }
1285
1286 sub verify_media_type {
1287     my ($opt, $vtype, $media) = @_;
1288
1289     return if !$media;
1290
1291     my $etype;
1292     if ($media eq 'disk') {
1293         $etype = 'images';
1294     } elsif ($media eq 'cdrom') {
1295         $etype = 'iso';
1296     } else {
1297         die "internal error";
1298     }
1299
1300     return if ($vtype eq $etype);
1301
1302     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1303 }
1304
1305 sub cleanup_drive_path {
1306     my ($opt, $storecfg, $drive) = @_;
1307
1308     # try to convert filesystem paths to volume IDs
1309
1310     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1311         ($drive->{file} !~ m|^/dev/.+|) &&
1312         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1313         ($drive->{file} !~ m/^\d+$/)) {
1314         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1315         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1316         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1317         verify_media_type($opt, $vtype, $drive->{media});
1318         $drive->{file} = $volid;
1319     }
1320
1321     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1322 }
1323
1324 sub parse_hotplug_features {
1325     my ($data) = @_;
1326
1327     my $res = {};
1328
1329     return $res if $data eq '0';
1330
1331     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1332
1333     foreach my $feature (PVE::Tools::split_list($data)) {
1334         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1335             $res->{$1} = 1;
1336         } else {
1337             die "invalid hotplug feature '$feature'\n";
1338         }
1339     }
1340     return $res;
1341 }
1342
1343 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1344 sub pve_verify_hotplug_features {
1345     my ($value, $noerr) = @_;
1346
1347     return $value if parse_hotplug_features($value);
1348
1349     return undef if $noerr;
1350
1351     die "unable to parse hotplug option\n";
1352 }
1353
1354 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1355 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1356 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1357 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1358 #        [,iothread=on][,serial=serial][,model=model]
1359
1360 sub parse_drive {
1361     my ($key, $data) = @_;
1362
1363     my ($interface, $index);
1364
1365     if ($key =~ m/^([^\d]+)(\d+)$/) {
1366         $interface = $1;
1367         $index = $2;
1368     } else {
1369         return undef;
1370     }
1371
1372     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1373                                      : $confdesc->{$key}->{format};
1374     if (!$desc) {
1375         warn "invalid drive key: $key\n";
1376         return undef;
1377     }
1378     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1379     return undef if !$res;
1380     $res->{interface} = $interface;
1381     $res->{index} = $index;
1382
1383     my $error = 0;
1384     foreach my $opt (qw(bps bps_rd bps_wr)) {
1385         if (my $bps = defined(delete $res->{$opt})) {
1386             if (defined($res->{"m$opt"})) {
1387                 warn "both $opt and m$opt specified\n";
1388                 ++$error;
1389                 next;
1390             }
1391             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1392         }
1393     }
1394
1395     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1396     for my $requirement (
1397         [mbps_max => 'mbps'],
1398         [mbps_rd_max => 'mbps_rd'],
1399         [mbps_wr_max => 'mbps_wr'],
1400         [miops_max => 'miops'],
1401         [miops_rd_max => 'miops_rd'],
1402         [miops_wr_max => 'miops_wr'],
1403         [bps_max_length => 'mbps_max'],
1404         [bps_rd_max_length => 'mbps_rd_max'],
1405         [bps_wr_max_length => 'mbps_wr_max'],
1406         [iops_max_length => 'iops_max'],
1407         [iops_rd_max_length => 'iops_rd_max'],
1408         [iops_wr_max_length => 'iops_wr_max']) {
1409         my ($option, $requires) = @$requirement;
1410         if ($res->{$option} && !$res->{$requires}) {
1411             warn "$option requires $requires\n";
1412             ++$error;
1413         }
1414     }
1415
1416     return undef if $error;
1417
1418     return undef if $res->{mbps_rd} && $res->{mbps};
1419     return undef if $res->{mbps_wr} && $res->{mbps};
1420     return undef if $res->{iops_rd} && $res->{iops};
1421     return undef if $res->{iops_wr} && $res->{iops};
1422
1423     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1424         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1425         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1426         return undef if $res->{interface} eq 'virtio';
1427     }
1428
1429     if (my $size = $res->{size}) {
1430         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1431     }
1432
1433     return $res;
1434 }
1435
1436 sub print_drive {
1437     my ($vmid, $drive) = @_;
1438     my $data = { %$drive };
1439     delete $data->{$_} for qw(index interface);
1440     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1441 }
1442
1443 sub scsi_inquiry {
1444     my($fh, $noerr) = @_;
1445
1446     my $SG_IO = 0x2285;
1447     my $SG_GET_VERSION_NUM = 0x2282;
1448
1449     my $versionbuf = "\x00" x 8;
1450     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1451     if (!$ret) {
1452         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1453         return undef;
1454     }
1455     my $version = unpack("I", $versionbuf);
1456     if ($version < 30000) {
1457         die "scsi generic interface too old\n"  if !$noerr;
1458         return undef;
1459     }
1460
1461     my $buf = "\x00" x 36;
1462     my $sensebuf = "\x00" x 8;
1463     my $cmd = pack("C x3 C x1", 0x12, 36);
1464
1465     # see /usr/include/scsi/sg.h
1466     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1467
1468     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1469                       length($sensebuf), 0, length($buf), $buf,
1470                       $cmd, $sensebuf, 6000);
1471
1472     $ret = ioctl($fh, $SG_IO, $packet);
1473     if (!$ret) {
1474         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1475         return undef;
1476     }
1477
1478     my @res = unpack($sg_io_hdr_t, $packet);
1479     if ($res[17] || $res[18]) {
1480         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1481         return undef;
1482     }
1483
1484     my $res = {};
1485     (my $byte0, my $byte1, $res->{vendor},
1486      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1487
1488     $res->{removable} = $byte1 & 128 ? 1 : 0;
1489     $res->{type} = $byte0 & 31;
1490
1491     return $res;
1492 }
1493
1494 sub path_is_scsi {
1495     my ($path) = @_;
1496
1497     my $fh = IO::File->new("+<$path") || return undef;
1498     my $res = scsi_inquiry($fh, 1);
1499     close($fh);
1500
1501     return $res;
1502 }
1503
1504 sub machine_type_is_q35 {
1505     my ($conf) = @_;
1506
1507     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1508 }
1509
1510 sub print_tabletdevice_full {
1511     my ($conf) = @_;
1512
1513     my $q35 = machine_type_is_q35($conf);
1514
1515     # we use uhci for old VMs because tablet driver was buggy in older qemu
1516     my $usbbus = $q35 ? "ehci" : "uhci";
1517
1518     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1519 }
1520
1521 sub print_drivedevice_full {
1522     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1523
1524     my $device = '';
1525     my $maxdev = 0;
1526
1527     if ($drive->{interface} eq 'virtio') {
1528         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1529         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1530         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1531     } elsif ($drive->{interface} eq 'scsi') {
1532
1533         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1534         my $unit = $drive->{index} % $maxdev;
1535         my $devicetype = 'hd';
1536         my $path = '';
1537         if (drive_is_cdrom($drive)) {
1538             $devicetype = 'cd';
1539         } else {
1540             if ($drive->{file} =~ m|^/|) {
1541                 $path = $drive->{file};
1542                 if (my $info = path_is_scsi($path)) {
1543                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1544                         $devicetype = 'block';
1545                     } elsif ($info->{type} == 1) { # tape
1546                         $devicetype = 'generic';
1547                     }
1548                 }
1549             } else {
1550                  $path = PVE::Storage::path($storecfg, $drive->{file});
1551             }
1552
1553             if($path =~ m/^iscsi\:\/\//){
1554                 $devicetype = 'generic';
1555             }
1556         }
1557
1558         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1559             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1560         } else {
1561             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1562         }
1563
1564     } elsif ($drive->{interface} eq 'ide'){
1565         $maxdev = 2;
1566         my $controller = int($drive->{index} / $maxdev);
1567         my $unit = $drive->{index} % $maxdev;
1568         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1569
1570         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1571         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1572             $model = URI::Escape::uri_unescape($model);
1573             $device .= ",model=$model";
1574         }
1575     } elsif ($drive->{interface} eq 'sata'){
1576         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1577         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1578         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1579     } elsif ($drive->{interface} eq 'usb') {
1580         die "implement me";
1581         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1582     } else {
1583         die "unsupported interface type";
1584     }
1585
1586     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1587
1588     return $device;
1589 }
1590
1591 sub get_initiator_name {
1592     my $initiator;
1593
1594     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1595     while (defined(my $line = <$fh>)) {
1596         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1597         $initiator = $1;
1598         last;
1599     }
1600     $fh->close();
1601
1602     return $initiator;
1603 }
1604
1605 sub print_drive_full {
1606     my ($storecfg, $vmid, $drive) = @_;
1607
1608     my $path;
1609     my $volid = $drive->{file};
1610     my $format;
1611
1612     if (drive_is_cdrom($drive)) {
1613         $path = get_iso_path($storecfg, $vmid, $volid);
1614     } else {
1615         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1616         if ($storeid) {
1617             $path = PVE::Storage::path($storecfg, $volid);
1618             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1619             $format = qemu_img_format($scfg, $volname);
1620         } else {
1621             $path = $volid;
1622             $format = "raw";
1623         }
1624    }
1625
1626     my $opts = '';
1627     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1628     foreach my $o (@qemu_drive_options) {
1629         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1630     }
1631
1632     # snapshot only accepts on|off
1633     if (defined($drive->{snapshot})) {
1634         my $v = $drive->{snapshot} ? 'on' : 'off';
1635         $opts .= ",snapshot=$v";
1636     }
1637
1638     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1639         my ($dir, $qmpname) = @$type;
1640         if (my $v = $drive->{"mbps$dir"}) {
1641             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1642         }
1643         if (my $v = $drive->{"mbps${dir}_max"}) {
1644             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1645         }
1646         if (my $v = $drive->{"bps${dir}_max_length"}) {
1647             $opts .= ",throttling.bps$qmpname-max-length=$v";
1648         }
1649         if (my $v = $drive->{"iops${dir}"}) {
1650             $opts .= ",throttling.iops$qmpname=$v";
1651         }
1652         if (my $v = $drive->{"iops${dir}_max"}) {
1653             $opts .= ",throttling.iops$qmpname-max=$v";
1654         }
1655         if (my $v = $drive->{"iops${dir}_max_length"}) {
1656             $opts .= ",throttling.iops$qmpname-max-length=$v";
1657         }
1658     }
1659
1660     if (my $serial = $drive->{serial}) {
1661         $serial = URI::Escape::uri_unescape($serial);
1662         $opts .= ",serial=$serial";
1663     }
1664
1665     $opts .= ",format=$format" if $format && !$drive->{format};
1666
1667     my $cache_direct = 0;
1668
1669     if (my $cache = $drive->{cache}) {
1670         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1671     } elsif (!drive_is_cdrom($drive)) {
1672         $opts .= ",cache=none";
1673         $cache_direct = 1;
1674     }
1675
1676     # aio native works only with O_DIRECT
1677     if (!$drive->{aio}) {
1678         if($cache_direct) {
1679             $opts .= ",aio=native";
1680         } else {
1681             $opts .= ",aio=threads";
1682         }
1683     }
1684
1685     if (!drive_is_cdrom($drive)) {
1686         my $detectzeroes;
1687         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1688             $detectzeroes = 'off';
1689         } elsif ($drive->{discard}) {
1690             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1691         } else {
1692             # This used to be our default with discard not being specified:
1693             $detectzeroes = 'on';
1694         }
1695         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1696     }
1697
1698     my $pathinfo = $path ? "file=$path," : '';
1699
1700     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1701 }
1702
1703 sub print_netdevice_full {
1704     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1705
1706     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1707
1708     my $device = $net->{model};
1709     if ($net->{model} eq 'virtio') {
1710          $device = 'virtio-net-pci';
1711      };
1712
1713     my $pciaddr = print_pci_addr("$netid", $bridges);
1714     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1715     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1716         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1717         my $vectors = $net->{queues} * 2 + 2;
1718         $tmpstr .= ",vectors=$vectors,mq=on";
1719     }
1720     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1721
1722     if ($use_old_bios_files) {
1723         my $romfile;
1724         if ($device eq 'virtio-net-pci') {
1725             $romfile = 'pxe-virtio.rom';
1726         } elsif ($device eq 'e1000') {
1727             $romfile = 'pxe-e1000.rom';
1728         } elsif ($device eq 'ne2k') {
1729             $romfile = 'pxe-ne2k_pci.rom';
1730         } elsif ($device eq 'pcnet') {
1731             $romfile = 'pxe-pcnet.rom';
1732         } elsif ($device eq 'rtl8139') {
1733             $romfile = 'pxe-rtl8139.rom';
1734         }
1735         $tmpstr .= ",romfile=$romfile" if $romfile;
1736     }
1737
1738     return $tmpstr;
1739 }
1740
1741 sub print_netdev_full {
1742     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1743
1744     my $i = '';
1745     if ($netid =~ m/^net(\d+)$/) {
1746         $i = int($1);
1747     }
1748
1749     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1750
1751     my $ifname = "tap${vmid}i$i";
1752
1753     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1754     die "interface name '$ifname' is too long (max 15 character)\n"
1755         if length($ifname) >= 16;
1756
1757     my $vhostparam = '';
1758     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1759
1760     my $vmname = $conf->{name} || "vm$vmid";
1761
1762     my $netdev = "";
1763     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1764
1765     if ($net->{bridge}) {
1766         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1767     } else {
1768         $netdev = "type=user,id=$netid,hostname=$vmname";
1769     }
1770
1771     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1772
1773     return $netdev;
1774 }
1775
1776
1777 sub print_cpu_device {
1778     my ($conf, $id) = @_;
1779
1780     my $kvm = $conf->{kvm} // 1;
1781     my $cpu = $kvm ? "kvm64" : "qemu64";
1782     if (my $cputype = $conf->{cpu}) {
1783         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1784             or die "Cannot parse cpu description: $cputype\n";
1785         $cpu = $cpuconf->{cputype};
1786     }
1787
1788     my $cores = $conf->{cores} || 1;
1789
1790     my $current_core = ($id - 1) % $cores;
1791     my $current_socket = int(($id - 1 - $current_core)/$cores);
1792
1793     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1794 }
1795
1796 sub drive_is_cdrom {
1797     my ($drive) = @_;
1798
1799     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1800
1801 }
1802
1803 sub parse_number_sets {
1804     my ($set) = @_;
1805     my $res = [];
1806     foreach my $part (split(/;/, $set)) {
1807         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1808             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1809             push @$res, [ $1, $2 ];
1810         } else {
1811             die "invalid range: $part\n";
1812         }
1813     }
1814     return $res;
1815 }
1816
1817 sub parse_numa {
1818     my ($data) = @_;
1819
1820     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1821     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1822     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1823     return $res;
1824 }
1825
1826 sub parse_hostpci {
1827     my ($value) = @_;
1828
1829     return undef if !$value;
1830
1831     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1832
1833     my @idlist = split(/;/, $res->{host});
1834     delete $res->{host};
1835     foreach my $id (@idlist) {
1836         if ($id =~ /^$PCIRE$/) {
1837             if (defined($2)) {
1838                 push @{$res->{pciid}}, { id => $1, function => $2 };
1839             } else {
1840                 my $pcidevices = lspci($1);
1841                 $res->{pciid} = $pcidevices->{$1};
1842             }
1843         } else {
1844             # should have been caught by parse_property_string already
1845             die "failed to parse PCI id: $id\n";
1846         }
1847     }
1848     return $res;
1849 }
1850
1851 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1852 sub parse_net {
1853     my ($data) = @_;
1854
1855     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1856     if ($@) {
1857         warn $@;
1858         return undef;
1859     }
1860     if (!defined($res->{macaddr})) {
1861         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1862         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1863     }
1864     return $res;
1865 }
1866
1867 sub print_net {
1868     my $net = shift;
1869
1870     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1871 }
1872
1873 sub add_random_macs {
1874     my ($settings) = @_;
1875
1876     foreach my $opt (keys %$settings) {
1877         next if $opt !~ m/^net(\d+)$/;
1878         my $net = parse_net($settings->{$opt});
1879         next if !$net;
1880         $settings->{$opt} = print_net($net);
1881     }
1882 }
1883
1884 sub vm_is_volid_owner {
1885     my ($storecfg, $vmid, $volid) = @_;
1886
1887     if ($volid !~  m|^/|) {
1888         my ($path, $owner);
1889         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1890         if ($owner && ($owner == $vmid)) {
1891             return 1;
1892         }
1893     }
1894
1895     return undef;
1896 }
1897
1898 sub split_flagged_list {
1899     my $text = shift || '';
1900     $text =~ s/[,;]/ /g;
1901     $text =~ s/^\s+//;
1902     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1903 }
1904
1905 sub join_flagged_list {
1906     my ($how, $lst) = @_;
1907     join $how, map { $lst->{$_} . $_ } keys %$lst;
1908 }
1909
1910 sub vmconfig_delete_pending_option {
1911     my ($conf, $key, $force) = @_;
1912
1913     delete $conf->{pending}->{$key};
1914     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1915     $pending_delete_hash->{$key} = $force ? '!' : '';
1916     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1917 }
1918
1919 sub vmconfig_undelete_pending_option {
1920     my ($conf, $key) = @_;
1921
1922     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1923     delete $pending_delete_hash->{$key};
1924
1925     if (%$pending_delete_hash) {
1926         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1927     } else {
1928         delete $conf->{pending}->{delete};
1929     }
1930 }
1931
1932 sub vmconfig_register_unused_drive {
1933     my ($storecfg, $vmid, $conf, $drive) = @_;
1934
1935     if (!drive_is_cdrom($drive)) {
1936         my $volid = $drive->{file};
1937         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1938             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1939         }
1940     }
1941 }
1942
1943 sub vmconfig_cleanup_pending {
1944     my ($conf) = @_;
1945
1946     # remove pending changes when nothing changed
1947     my $changes;
1948     foreach my $opt (keys %{$conf->{pending}}) {
1949         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1950             $changes = 1;
1951             delete $conf->{pending}->{$opt};
1952         }
1953     }
1954
1955     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1956     my $pending_delete_hash = {};
1957     while (my ($opt, $force) = each %$current_delete_hash) {
1958         if (defined($conf->{$opt})) {
1959             $pending_delete_hash->{$opt} = $force;
1960         } else {
1961             $changes = 1;
1962         }
1963     }
1964
1965     if (%$pending_delete_hash) {
1966         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1967     } else {
1968         delete $conf->{pending}->{delete};
1969     }
1970
1971     return $changes;
1972 }
1973
1974 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1975 my $smbios1_fmt = {
1976     uuid => {
1977         type => 'string',
1978         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1979         format_description => 'UUID',
1980         description => "Set SMBIOS1 UUID.",
1981         optional => 1,
1982     },
1983     version => {
1984         type => 'string',
1985         pattern => '\S+',
1986         format_description => 'string',
1987         description => "Set SMBIOS1 version.",
1988         optional => 1,
1989     },
1990     serial => {
1991         type => 'string',
1992         pattern => '\S+',
1993         format_description => 'string',
1994         description => "Set SMBIOS1 serial number.",
1995         optional => 1,
1996     },
1997     manufacturer => {
1998         type => 'string',
1999         pattern => '\S+',
2000         format_description => 'string',
2001         description => "Set SMBIOS1 manufacturer.",
2002         optional => 1,
2003     },
2004     product => {
2005         type => 'string',
2006         pattern => '\S+',
2007         format_description => 'string',
2008         description => "Set SMBIOS1 product ID.",
2009         optional => 1,
2010     },
2011     sku => {
2012         type => 'string',
2013         pattern => '\S+',
2014         format_description => 'string',
2015         description => "Set SMBIOS1 SKU string.",
2016         optional => 1,
2017     },
2018     family => {
2019         type => 'string',
2020         pattern => '\S+',
2021         format_description => 'string',
2022         description => "Set SMBIOS1 family string.",
2023         optional => 1,
2024     },
2025 };
2026
2027 sub parse_smbios1 {
2028     my ($data) = @_;
2029
2030     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2031     warn $@ if $@;
2032     return $res;
2033 }
2034
2035 sub print_smbios1 {
2036     my ($smbios1) = @_;
2037     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2038 }
2039
2040 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2041
2042 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2043 sub verify_bootdisk {
2044     my ($value, $noerr) = @_;
2045
2046     return $value if is_valid_drivename($value);
2047
2048     return undef if $noerr;
2049
2050     die "invalid boot disk '$value'\n";
2051 }
2052
2053 sub parse_watchdog {
2054     my ($value) = @_;
2055
2056     return undef if !$value;
2057
2058     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2059     warn $@ if $@;
2060     return $res;
2061 }
2062
2063 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2064 sub verify_usb_device {
2065     my ($value, $noerr) = @_;
2066
2067     return $value if parse_usb_device($value);
2068
2069     return undef if $noerr;
2070
2071     die "unable to parse usb device\n";
2072 }
2073
2074 # add JSON properties for create and set function
2075 sub json_config_properties {
2076     my $prop = shift;
2077
2078     foreach my $opt (keys %$confdesc) {
2079         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2080         $prop->{$opt} = $confdesc->{$opt};
2081     }
2082
2083     return $prop;
2084 }
2085
2086 sub check_type {
2087     my ($key, $value) = @_;
2088
2089     die "unknown setting '$key'\n" if !$confdesc->{$key};
2090
2091     my $type = $confdesc->{$key}->{type};
2092
2093     if (!defined($value)) {
2094         die "got undefined value\n";
2095     }
2096
2097     if ($value =~ m/[\n\r]/) {
2098         die "property contains a line feed\n";
2099     }
2100
2101     if ($type eq 'boolean') {
2102         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2103         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2104         die "type check ('boolean') failed - got '$value'\n";
2105     } elsif ($type eq 'integer') {
2106         return int($1) if $value =~ m/^(\d+)$/;
2107         die "type check ('integer') failed - got '$value'\n";
2108     } elsif ($type eq 'number') {
2109         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2110         die "type check ('number') failed - got '$value'\n";
2111     } elsif ($type eq 'string') {
2112         if (my $fmt = $confdesc->{$key}->{format}) {
2113             PVE::JSONSchema::check_format($fmt, $value);
2114             return $value;
2115         }
2116         $value =~ s/^\"(.*)\"$/$1/;
2117         return $value;
2118     } else {
2119         die "internal error"
2120     }
2121 }
2122
2123 sub check_iommu_support{
2124     #fixme : need to check IOMMU support
2125     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2126
2127     my $iommu=1;
2128     return $iommu;
2129
2130 }
2131
2132 sub touch_config {
2133     my ($vmid) = @_;
2134
2135     my $conf = PVE::QemuConfig->config_file($vmid);
2136     utime undef, undef, $conf;
2137 }
2138
2139 sub destroy_vm {
2140     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2141
2142     my $conffile = PVE::QemuConfig->config_file($vmid);
2143
2144     my $conf = PVE::QemuConfig->load_config($vmid);
2145
2146     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2147
2148     if ($conf->{template}) {
2149         # check if any base image is still used by a linked clone
2150         foreach_drive($conf, sub {
2151                 my ($ds, $drive) = @_;
2152
2153                 return if drive_is_cdrom($drive);
2154
2155                 my $volid = $drive->{file};
2156
2157                 return if !$volid || $volid =~ m|^/|;
2158
2159                 die "base volume '$volid' is still in use by linked cloned\n"
2160                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2161
2162         });
2163     }
2164
2165     # only remove disks owned by this VM
2166     foreach_drive($conf, sub {
2167         my ($ds, $drive) = @_;
2168
2169         return if drive_is_cdrom($drive);
2170
2171         my $volid = $drive->{file};
2172
2173         return if !$volid || $volid =~ m|^/|;
2174
2175         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2176         return if !$path || !$owner || ($owner != $vmid);
2177
2178         eval {
2179             PVE::Storage::vdisk_free($storecfg, $volid);
2180         };
2181         warn "Could not remove disk '$volid', check manually: $@" if $@;
2182
2183     });
2184
2185     if ($keep_empty_config) {
2186         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2187     } else {
2188         unlink $conffile;
2189     }
2190
2191     # also remove unused disk
2192     eval {
2193         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2194
2195         eval {
2196             PVE::Storage::foreach_volid($dl, sub {
2197                 my ($volid, $sid, $volname, $d) = @_;
2198                 PVE::Storage::vdisk_free($storecfg, $volid);
2199             });
2200         };
2201         warn $@ if $@;
2202
2203     };
2204     warn $@ if $@;
2205 }
2206
2207 sub parse_vm_config {
2208     my ($filename, $raw) = @_;
2209
2210     return undef if !defined($raw);
2211
2212     my $res = {
2213         digest => Digest::SHA::sha1_hex($raw),
2214         snapshots => {},
2215         pending => {},
2216     };
2217
2218     $filename =~ m|/qemu-server/(\d+)\.conf$|
2219         || die "got strange filename '$filename'";
2220
2221     my $vmid = $1;
2222
2223     my $conf = $res;
2224     my $descr;
2225     my $section = '';
2226
2227     my @lines = split(/\n/, $raw);
2228     foreach my $line (@lines) {
2229         next if $line =~ m/^\s*$/;
2230
2231         if ($line =~ m/^\[PENDING\]\s*$/i) {
2232             $section = 'pending';
2233             if (defined($descr)) {
2234                 $descr =~ s/\s+$//;
2235                 $conf->{description} = $descr;
2236             }
2237             $descr = undef;
2238             $conf = $res->{$section} = {};
2239             next;
2240
2241         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2242             $section = $1;
2243             if (defined($descr)) {
2244                 $descr =~ s/\s+$//;
2245                 $conf->{description} = $descr;
2246             }
2247             $descr = undef;
2248             $conf = $res->{snapshots}->{$section} = {};
2249             next;
2250         }
2251
2252         if ($line =~ m/^\#(.*)\s*$/) {
2253             $descr = '' if !defined($descr);
2254             $descr .= PVE::Tools::decode_text($1) . "\n";
2255             next;
2256         }
2257
2258         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2259             $descr = '' if !defined($descr);
2260             $descr .= PVE::Tools::decode_text($2);
2261         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2262             $conf->{snapstate} = $1;
2263         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2264             my $key = $1;
2265             my $value = $2;
2266             $conf->{$key} = $value;
2267         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2268             my $value = $1;
2269             if ($section eq 'pending') {
2270                 $conf->{delete} = $value; # we parse this later
2271             } else {
2272                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2273             }
2274         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2275             my $key = $1;
2276             my $value = $2;
2277             eval { $value = check_type($key, $value); };
2278             if ($@) {
2279                 warn "vm $vmid - unable to parse value of '$key' - $@";
2280             } else {
2281                 $key = 'ide2' if $key eq 'cdrom';
2282                 my $fmt = $confdesc->{$key}->{format};
2283                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2284                     my $v = parse_drive($key, $value);
2285                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2286                         $v->{file} = $volid;
2287                         $value = print_drive($vmid, $v);
2288                     } else {
2289                         warn "vm $vmid - unable to parse value of '$key'\n";
2290                         next;
2291                     }
2292                 }
2293
2294                 $conf->{$key} = $value;
2295             }
2296         }
2297     }
2298
2299     if (defined($descr)) {
2300         $descr =~ s/\s+$//;
2301         $conf->{description} = $descr;
2302     }
2303     delete $res->{snapstate}; # just to be sure
2304
2305     return $res;
2306 }
2307
2308 sub write_vm_config {
2309     my ($filename, $conf) = @_;
2310
2311     delete $conf->{snapstate}; # just to be sure
2312
2313     if ($conf->{cdrom}) {
2314         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2315         $conf->{ide2} = $conf->{cdrom};
2316         delete $conf->{cdrom};
2317     }
2318
2319     # we do not use 'smp' any longer
2320     if ($conf->{sockets}) {
2321         delete $conf->{smp};
2322     } elsif ($conf->{smp}) {
2323         $conf->{sockets} = $conf->{smp};
2324         delete $conf->{cores};
2325         delete $conf->{smp};
2326     }
2327
2328     my $used_volids = {};
2329
2330     my $cleanup_config = sub {
2331         my ($cref, $pending, $snapname) = @_;
2332
2333         foreach my $key (keys %$cref) {
2334             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2335                 $key eq 'snapstate' || $key eq 'pending';
2336             my $value = $cref->{$key};
2337             if ($key eq 'delete') {
2338                 die "propertry 'delete' is only allowed in [PENDING]\n"
2339                     if !$pending;
2340                 # fixme: check syntax?
2341                 next;
2342             }
2343             eval { $value = check_type($key, $value); };
2344             die "unable to parse value of '$key' - $@" if $@;
2345
2346             $cref->{$key} = $value;
2347
2348             if (!$snapname && is_valid_drivename($key)) {
2349                 my $drive = parse_drive($key, $value);
2350                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2351             }
2352         }
2353     };
2354
2355     &$cleanup_config($conf);
2356
2357     &$cleanup_config($conf->{pending}, 1);
2358
2359     foreach my $snapname (keys %{$conf->{snapshots}}) {
2360         die "internal error" if $snapname eq 'pending';
2361         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2362     }
2363
2364     # remove 'unusedX' settings if we re-add a volume
2365     foreach my $key (keys %$conf) {
2366         my $value = $conf->{$key};
2367         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2368             delete $conf->{$key};
2369         }
2370     }
2371
2372     my $generate_raw_config = sub {
2373         my ($conf, $pending) = @_;
2374
2375         my $raw = '';
2376
2377         # add description as comment to top of file
2378         if (defined(my $descr = $conf->{description})) {
2379             if ($descr) {
2380                 foreach my $cl (split(/\n/, $descr)) {
2381                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2382                 }
2383             } else {
2384                 $raw .= "#\n" if $pending;
2385             }
2386         }
2387
2388         foreach my $key (sort keys %$conf) {
2389             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2390             $raw .= "$key: $conf->{$key}\n";
2391         }
2392         return $raw;
2393     };
2394
2395     my $raw = &$generate_raw_config($conf);
2396
2397     if (scalar(keys %{$conf->{pending}})){
2398         $raw .= "\n[PENDING]\n";
2399         $raw .= &$generate_raw_config($conf->{pending}, 1);
2400     }
2401
2402     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2403         $raw .= "\n[$snapname]\n";
2404         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2405     }
2406
2407     return $raw;
2408 }
2409
2410 sub load_defaults {
2411
2412     my $res = {};
2413
2414     # we use static defaults from our JSON schema configuration
2415     foreach my $key (keys %$confdesc) {
2416         if (defined(my $default = $confdesc->{$key}->{default})) {
2417             $res->{$key} = $default;
2418         }
2419     }
2420
2421     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2422     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2423
2424     return $res;
2425 }
2426
2427 sub config_list {
2428     my $vmlist = PVE::Cluster::get_vmlist();
2429     my $res = {};
2430     return $res if !$vmlist || !$vmlist->{ids};
2431     my $ids = $vmlist->{ids};
2432
2433     foreach my $vmid (keys %$ids) {
2434         my $d = $ids->{$vmid};
2435         next if !$d->{node} || $d->{node} ne $nodename;
2436         next if !$d->{type} || $d->{type} ne 'qemu';
2437         $res->{$vmid}->{exists} = 1;
2438     }
2439     return $res;
2440 }
2441
2442 # test if VM uses local resources (to prevent migration)
2443 sub check_local_resources {
2444     my ($conf, $noerr) = @_;
2445
2446     my $loc_res = 0;
2447
2448     $loc_res = 1 if $conf->{hostusb}; # old syntax
2449     $loc_res = 1 if $conf->{hostpci}; # old syntax
2450
2451     foreach my $k (keys %$conf) {
2452         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2453         # sockets are safe: they will recreated be on the target side post-migrate
2454         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2455         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2456     }
2457
2458     die "VM uses local resources\n" if $loc_res && !$noerr;
2459
2460     return $loc_res;
2461 }
2462
2463 # check if used storages are available on all nodes (use by migrate)
2464 sub check_storage_availability {
2465     my ($storecfg, $conf, $node) = @_;
2466
2467     foreach_drive($conf, sub {
2468         my ($ds, $drive) = @_;
2469
2470         my $volid = $drive->{file};
2471         return if !$volid;
2472
2473         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2474         return if !$sid;
2475
2476         # check if storage is available on both nodes
2477         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2478         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2479    });
2480 }
2481
2482 # list nodes where all VM images are available (used by has_feature API)
2483 sub shared_nodes {
2484     my ($conf, $storecfg) = @_;
2485
2486     my $nodelist = PVE::Cluster::get_nodelist();
2487     my $nodehash = { map { $_ => 1 } @$nodelist };
2488     my $nodename = PVE::INotify::nodename();
2489
2490     foreach_drive($conf, sub {
2491         my ($ds, $drive) = @_;
2492
2493         my $volid = $drive->{file};
2494         return if !$volid;
2495
2496         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2497         if ($storeid) {
2498             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2499             if ($scfg->{disable}) {
2500                 $nodehash = {};
2501             } elsif (my $avail = $scfg->{nodes}) {
2502                 foreach my $node (keys %$nodehash) {
2503                     delete $nodehash->{$node} if !$avail->{$node};
2504                 }
2505             } elsif (!$scfg->{shared}) {
2506                 foreach my $node (keys %$nodehash) {
2507                     delete $nodehash->{$node} if $node ne $nodename
2508                 }
2509             }
2510         }
2511     });
2512
2513     return $nodehash
2514 }
2515
2516 sub check_cmdline {
2517     my ($pidfile, $pid) = @_;
2518
2519     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2520     if (defined($fh)) {
2521         my $line = <$fh>;
2522         $fh->close;
2523         return undef if !$line;
2524         my @param = split(/\0/, $line);
2525
2526         my $cmd = $param[0];
2527         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2528
2529         for (my $i = 0; $i < scalar (@param); $i++) {
2530             my $p = $param[$i];
2531             next if !$p;
2532             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2533                 my $p = $param[$i+1];
2534                 return 1 if $p && ($p eq $pidfile);
2535                 return undef;
2536             }
2537         }
2538     }
2539     return undef;
2540 }
2541
2542 sub check_running {
2543     my ($vmid, $nocheck, $node) = @_;
2544
2545     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2546
2547     die "unable to find configuration file for VM $vmid - no such machine\n"
2548         if !$nocheck && ! -f $filename;
2549
2550     my $pidfile = pidfile_name($vmid);
2551
2552     if (my $fd = IO::File->new("<$pidfile")) {
2553         my $st = stat($fd);
2554         my $line = <$fd>;
2555         close($fd);
2556
2557         my $mtime = $st->mtime;
2558         if ($mtime > time()) {
2559             warn "file '$filename' modified in future\n";
2560         }
2561
2562         if ($line =~ m/^(\d+)$/) {
2563             my $pid = $1;
2564             if (check_cmdline($pidfile, $pid)) {
2565                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2566                     return $pid;
2567                 }
2568             }
2569         }
2570     }
2571
2572     return undef;
2573 }
2574
2575 sub vzlist {
2576
2577     my $vzlist = config_list();
2578
2579     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2580
2581     while (defined(my $de = $fd->read)) {
2582         next if $de !~ m/^(\d+)\.pid$/;
2583         my $vmid = $1;
2584         next if !defined($vzlist->{$vmid});
2585         if (my $pid = check_running($vmid)) {
2586             $vzlist->{$vmid}->{pid} = $pid;
2587         }
2588     }
2589
2590     return $vzlist;
2591 }
2592
2593 sub disksize {
2594     my ($storecfg, $conf) = @_;
2595
2596     my $bootdisk = $conf->{bootdisk};
2597     return undef if !$bootdisk;
2598     return undef if !is_valid_drivename($bootdisk);
2599
2600     return undef if !$conf->{$bootdisk};
2601
2602     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2603     return undef if !defined($drive);
2604
2605     return undef if drive_is_cdrom($drive);
2606
2607     my $volid = $drive->{file};
2608     return undef if !$volid;
2609
2610     return $drive->{size};
2611 }
2612
2613 my $last_proc_pid_stat;
2614
2615 # get VM status information
2616 # This must be fast and should not block ($full == false)
2617 # We only query KVM using QMP if $full == true (this can be slow)
2618 sub vmstatus {
2619     my ($opt_vmid, $full) = @_;
2620
2621     my $res = {};
2622
2623     my $storecfg = PVE::Storage::config();
2624
2625     my $list = vzlist();
2626     my $defaults = load_defaults();
2627
2628     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2629
2630     my $cpucount = $cpuinfo->{cpus} || 1;
2631
2632     foreach my $vmid (keys %$list) {
2633         next if $opt_vmid && ($vmid ne $opt_vmid);
2634
2635         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2636         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2637
2638         my $d = {};
2639         $d->{pid} = $list->{$vmid}->{pid};
2640
2641         # fixme: better status?
2642         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2643
2644         my $size = disksize($storecfg, $conf);
2645         if (defined($size)) {
2646             $d->{disk} = 0; # no info available
2647             $d->{maxdisk} = $size;
2648         } else {
2649             $d->{disk} = 0;
2650             $d->{maxdisk} = 0;
2651         }
2652
2653         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2654             * ($conf->{cores} || $defaults->{cores});
2655         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2656         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2657
2658         $d->{name} = $conf->{name} || "VM $vmid";
2659         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2660             : $defaults->{memory}*(1024*1024);
2661
2662         if ($conf->{balloon}) {
2663             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2664             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2665                 : $defaults->{shares};
2666         }
2667
2668         $d->{uptime} = 0;
2669         $d->{cpu} = 0;
2670         $d->{mem} = 0;
2671
2672         $d->{netout} = 0;
2673         $d->{netin} = 0;
2674
2675         $d->{diskread} = 0;
2676         $d->{diskwrite} = 0;
2677
2678         $d->{template} = PVE::QemuConfig->is_template($conf);
2679
2680         $d->{serial} = 1 if conf_has_serial($conf);
2681
2682         $res->{$vmid} = $d;
2683     }
2684
2685     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2686     foreach my $dev (keys %$netdev) {
2687         next if $dev !~ m/^tap([1-9]\d*)i/;
2688         my $vmid = $1;
2689         my $d = $res->{$vmid};
2690         next if !$d;
2691
2692         $d->{netout} += $netdev->{$dev}->{receive};
2693         $d->{netin} += $netdev->{$dev}->{transmit};
2694
2695         if ($full) {
2696             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2697             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2698         }
2699
2700     }
2701
2702     my $ctime = gettimeofday;
2703
2704     foreach my $vmid (keys %$list) {
2705
2706         my $d = $res->{$vmid};
2707         my $pid = $d->{pid};
2708         next if !$pid;
2709
2710         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2711         next if !$pstat; # not running
2712
2713         my $used = $pstat->{utime} + $pstat->{stime};
2714
2715         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2716
2717         if ($pstat->{vsize}) {
2718             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2719         }
2720
2721         my $old = $last_proc_pid_stat->{$pid};
2722         if (!$old) {
2723             $last_proc_pid_stat->{$pid} = {
2724                 time => $ctime,
2725                 used => $used,
2726                 cpu => 0,
2727             };
2728             next;
2729         }
2730
2731         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2732
2733         if ($dtime > 1000) {
2734             my $dutime = $used -  $old->{used};
2735
2736             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2737             $last_proc_pid_stat->{$pid} = {
2738                 time => $ctime,
2739                 used => $used,
2740                 cpu => $d->{cpu},
2741             };
2742         } else {
2743             $d->{cpu} = $old->{cpu};
2744         }
2745     }
2746
2747     return $res if !$full;
2748
2749     my $qmpclient = PVE::QMPClient->new();
2750
2751     my $ballooncb = sub {
2752         my ($vmid, $resp) = @_;
2753
2754         my $info = $resp->{'return'};
2755         return if !$info->{max_mem};
2756
2757         my $d = $res->{$vmid};
2758
2759         # use memory assigned to VM
2760         $d->{maxmem} = $info->{max_mem};
2761         $d->{balloon} = $info->{actual};
2762
2763         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2764             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2765             $d->{freemem} = $info->{free_mem};
2766         }
2767
2768         $d->{ballooninfo} = $info;
2769     };
2770
2771     my $blockstatscb = sub {
2772         my ($vmid, $resp) = @_;
2773         my $data = $resp->{'return'} || [];
2774         my $totalrdbytes = 0;
2775         my $totalwrbytes = 0;
2776
2777         for my $blockstat (@$data) {
2778             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2779             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2780
2781             $blockstat->{device} =~ s/drive-//;
2782             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2783         }
2784         $res->{$vmid}->{diskread} = $totalrdbytes;
2785         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2786     };
2787
2788     my $statuscb = sub {
2789         my ($vmid, $resp) = @_;
2790
2791         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2792         # this fails if ballon driver is not loaded, so this must be
2793         # the last commnand (following command are aborted if this fails).
2794         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2795
2796         my $status = 'unknown';
2797         if (!defined($status = $resp->{'return'}->{status})) {
2798             warn "unable to get VM status\n";
2799             return;
2800         }
2801
2802         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2803     };
2804
2805     foreach my $vmid (keys %$list) {
2806         next if $opt_vmid && ($vmid ne $opt_vmid);
2807         next if !$res->{$vmid}->{pid}; # not running
2808         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2809     }
2810
2811     $qmpclient->queue_execute(undef, 2);
2812
2813     foreach my $vmid (keys %$list) {
2814         next if $opt_vmid && ($vmid ne $opt_vmid);
2815         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2816     }
2817
2818     return $res;
2819 }
2820
2821 sub foreach_drive {
2822     my ($conf, $func, @param) = @_;
2823
2824     foreach my $ds (valid_drive_names()) {
2825         next if !defined($conf->{$ds});
2826
2827         my $drive = parse_drive($ds, $conf->{$ds});
2828         next if !$drive;
2829
2830         &$func($ds, $drive, @param);
2831     }
2832 }
2833
2834 sub foreach_volid {
2835     my ($conf, $func, @param) = @_;
2836
2837     my $volhash = {};
2838
2839     my $test_volid = sub {
2840         my ($volid, $is_cdrom, $replicate, $snapname) = @_;
2841
2842         return if !$volid;
2843
2844         $volhash->{$volid}->{cdrom} //= 1;
2845         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2846
2847         $volhash->{$volid}->{replicate} //= 0;
2848         $volhash->{$volid}->{replicate} = 1 if $replicate;
2849
2850         $volhash->{$volid}->{referenced_in_config} //= 0;
2851         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
2852
2853         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
2854             if defined($snapname);
2855     };
2856
2857     foreach_drive($conf, sub {
2858         my ($ds, $drive) = @_;
2859         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, undef);
2860     });
2861
2862     foreach my $snapname (keys %{$conf->{snapshots}}) {
2863         my $snap = $conf->{snapshots}->{$snapname};
2864         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
2865         foreach_drive($snap, sub {
2866             my ($ds, $drive) = @_;
2867             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $snapname);
2868         });
2869     }
2870
2871     foreach my $volid (keys %$volhash) {
2872         &$func($volid, $volhash->{$volid}, @param);
2873     }
2874 }
2875
2876 sub conf_has_serial {
2877     my ($conf) = @_;
2878
2879     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2880         if ($conf->{"serial$i"}) {
2881             return 1;
2882         }
2883     }
2884
2885     return 0;
2886 }
2887
2888 sub vga_conf_has_spice {
2889     my ($vga) = @_;
2890
2891     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2892
2893     return $1 || 1;
2894 }
2895
2896 sub config_to_command {
2897     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2898
2899     my $cmd = [];
2900     my $globalFlags = [];
2901     my $machineFlags = [];
2902     my $rtcFlags = [];
2903     my $cpuFlags = [];
2904     my $devices = [];
2905     my $pciaddr = '';
2906     my $bridges = {};
2907     my $kvmver = kvm_user_version();
2908     my $vernum = 0; # unknown
2909     my $ostype = $conf->{ostype};
2910     my $winversion = windows_version($ostype);
2911     my $kvm = $conf->{kvm} // 1;
2912
2913     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
2914
2915     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2916         $vernum = $1*1000000+$2*1000;
2917     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2918         $vernum = $1*1000000+$2*1000+$3;
2919     }
2920
2921     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2922
2923     my $have_ovz = -f '/proc/vz/vestat';
2924
2925     my $q35 = machine_type_is_q35($conf);
2926     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2927     my $machine_type = $forcemachine || $conf->{machine};
2928     my $use_old_bios_files = undef;
2929     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2930
2931     my $cpuunits = defined($conf->{cpuunits}) ?
2932             $conf->{cpuunits} : $defaults->{cpuunits};
2933
2934     push @$cmd, '/usr/bin/kvm';
2935
2936     push @$cmd, '-id', $vmid;
2937
2938     my $use_virtio = 0;
2939
2940     my $qmpsocket = qmp_socket($vmid);
2941     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2942     push @$cmd, '-mon', "chardev=qmp,mode=control";
2943
2944
2945     push @$cmd, '-pidfile' , pidfile_name($vmid);
2946
2947     push @$cmd, '-daemonize';
2948
2949     if ($conf->{smbios1}) {
2950         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2951     }
2952
2953     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2954         die "uefi base image not found\n" if ! -f $OVMF_CODE;
2955
2956         my $path;
2957         my $format;
2958         if (my $efidisk = $conf->{efidisk0}) {
2959             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
2960             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2961             $format = $d->{format};
2962             if ($storeid) {
2963                 $path = PVE::Storage::path($storecfg, $d->{file});
2964                 if (!defined($format)) {
2965                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2966                     $format = qemu_img_format($scfg, $volname);
2967                 }
2968             } else {
2969                 $path = $d->{file};
2970                 die "efidisk format must be specified\n"
2971                     if !defined($format);
2972             }
2973         } else {
2974             warn "no efidisk configured! Using temporary efivars disk.\n";
2975             $path = "/tmp/$vmid-ovmf.fd";
2976             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
2977             $format = 'raw';
2978         }
2979
2980         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
2981         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
2982     }
2983
2984
2985     # add usb controllers
2986     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2987     push @$devices, @usbcontrollers if @usbcontrollers;
2988     my $vga = $conf->{vga};
2989
2990     my $qxlnum = vga_conf_has_spice($vga);
2991     $vga = 'qxl' if $qxlnum;
2992
2993     if (!$vga) {
2994         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
2995             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
2996         } else {
2997             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
2998         }
2999     }
3000
3001     # enable absolute mouse coordinates (needed by vnc)
3002     my $tablet;
3003     if (defined($conf->{tablet})) {
3004         $tablet = $conf->{tablet};
3005     } else {
3006         $tablet = $defaults->{tablet};
3007         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3008         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3009     }
3010
3011     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3012
3013     my $kvm_off = 0;
3014     my $gpu_passthrough;
3015
3016     # host pci devices
3017     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3018         my $d = parse_hostpci($conf->{"hostpci$i"});
3019         next if !$d;
3020
3021         my $pcie = $d->{pcie};
3022         if($pcie){
3023             die "q35 machine model is not enabled" if !$q35;
3024             $pciaddr = print_pcie_addr("hostpci$i");
3025         }else{
3026             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3027         }
3028
3029         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3030         my $romfile = $d->{romfile};
3031
3032         my $xvga = '';
3033         if ($d->{'x-vga'}) {
3034             $xvga = ',x-vga=on';
3035             $kvm_off = 1;
3036             $vga = 'none';
3037             $gpu_passthrough = 1;
3038
3039             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3040                 $xvga = "";
3041             }
3042         }
3043         my $pcidevices = $d->{pciid};
3044         my $multifunction = 1 if @$pcidevices > 1;
3045
3046         my $j=0;
3047         foreach my $pcidevice (@$pcidevices) {
3048
3049             my $id = "hostpci$i";
3050             $id .= ".$j" if $multifunction;
3051             my $addr = $pciaddr;
3052             $addr .= ".$j" if $multifunction;
3053             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3054
3055             if($j == 0){
3056                 $devicestr .= "$rombar$xvga";
3057                 $devicestr .= ",multifunction=on" if $multifunction;
3058                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3059             }
3060
3061             push @$devices, '-device', $devicestr;
3062             $j++;
3063         }
3064     }
3065
3066     # usb devices
3067     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3068     push @$devices, @usbdevices if @usbdevices;
3069     # serial devices
3070     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3071         if (my $path = $conf->{"serial$i"}) {
3072             if ($path eq 'socket') {
3073                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3074                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3075                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3076             } else {
3077                 die "no such serial device\n" if ! -c $path;
3078                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3079                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3080             }
3081         }
3082     }
3083
3084     # parallel devices
3085     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3086         if (my $path = $conf->{"parallel$i"}) {
3087             die "no such parallel device\n" if ! -c $path;
3088             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3089             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3090             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3091         }
3092     }
3093
3094     my $vmname = $conf->{name} || "vm$vmid";
3095
3096     push @$cmd, '-name', $vmname;
3097
3098     my $sockets = 1;
3099     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3100     $sockets = $conf->{sockets} if  $conf->{sockets};
3101
3102     my $cores = $conf->{cores} || 1;
3103
3104     my $maxcpus = $sockets * $cores;
3105
3106     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3107
3108     my $allowed_vcpus = $cpuinfo->{cpus};
3109
3110     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3111         if ($allowed_vcpus < $maxcpus);
3112
3113     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3114
3115         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3116         for (my $i = 2; $i <= $vcpus; $i++)  {
3117             my $cpustr = print_cpu_device($conf,$i);
3118             push @$cmd, '-device', $cpustr;
3119         }
3120
3121     } else {
3122
3123         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3124     }
3125     push @$cmd, '-nodefaults';
3126
3127     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3128
3129     my $bootindex_hash = {};
3130     my $i = 1;
3131     foreach my $o (split(//, $bootorder)) {
3132         $bootindex_hash->{$o} = $i*100;
3133         $i++;
3134     }
3135
3136     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3137
3138     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3139
3140     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3141
3142     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3143
3144     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3145         my $socket = vnc_socket($vmid);
3146         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3147     } else {
3148         push @$cmd, '-nographic';
3149     }
3150
3151     # time drift fix
3152     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3153
3154     my $useLocaltime = $conf->{localtime};
3155
3156     if ($winversion >= 5) { # windows
3157         $useLocaltime = 1 if !defined($conf->{localtime});
3158
3159         # use time drift fix when acpi is enabled
3160         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3161             $tdf = 1 if !defined($conf->{tdf});
3162         }
3163     }
3164
3165     if ($winversion >= 6) {
3166         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3167         push @$cmd, '-no-hpet';
3168     }
3169
3170     push @$rtcFlags, 'driftfix=slew' if $tdf;
3171
3172     if (!$kvm) {
3173         push @$machineFlags, 'accel=tcg';
3174     }
3175
3176     if ($machine_type) {
3177         push @$machineFlags, "type=${machine_type}";
3178     }
3179
3180     if ($conf->{startdate}) {
3181         push @$rtcFlags, "base=$conf->{startdate}";
3182     } elsif ($useLocaltime) {
3183         push @$rtcFlags, 'base=localtime';
3184     }
3185
3186     my $cpu = $kvm ? "kvm64" : "qemu64";
3187     if (my $cputype = $conf->{cpu}) {
3188         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3189             or die "Cannot parse cpu description: $cputype\n";
3190         $cpu = $cpuconf->{cputype};
3191         $kvm_off = 1 if $cpuconf->{hidden};
3192
3193         if (defined(my $flags = $cpuconf->{flags})) {
3194             push @$cpuFlags, split(";", $flags);
3195         }
3196     }
3197
3198     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3199
3200     push @$cpuFlags , '-x2apic'
3201         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3202
3203     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3204
3205     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3206
3207     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3208
3209         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3210         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3211     }
3212
3213     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3214
3215     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3216
3217     push @$cpuFlags, 'kvm=off' if $kvm_off;
3218
3219     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3220         die "internal error"; # should not happen
3221
3222     push @$cpuFlags, "vendor=${cpu_vendor}"
3223         if $cpu_vendor ne 'default';
3224
3225     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3226
3227     push @$cmd, '-cpu', $cpu;
3228
3229     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3230
3231     push @$cmd, '-S' if $conf->{freeze};
3232
3233     # set keyboard layout
3234     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3235     push @$cmd, '-k', $kb if $kb;
3236
3237     # enable sound
3238     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3239     #push @$cmd, '-soundhw', 'es1370';
3240     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3241
3242     if($conf->{agent}) {
3243         my $qgasocket = qmp_socket($vmid, 1);
3244         my $pciaddr = print_pci_addr("qga0", $bridges);
3245         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3246         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3247         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3248     }
3249
3250     my $spice_port;
3251
3252     if ($qxlnum) {
3253         if ($qxlnum > 1) {
3254             if ($winversion){
3255                 for(my $i = 1; $i < $qxlnum; $i++){
3256                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3257                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3258                 }
3259             } else {
3260                 # assume other OS works like Linux
3261                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3262                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3263             }
3264         }
3265
3266         my $pciaddr = print_pci_addr("spice", $bridges);
3267
3268         my $nodename = PVE::INotify::nodename();
3269         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3270         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3271         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3272         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3273         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3274
3275         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3276
3277         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3278         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3279         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3280     }
3281
3282     # enable balloon by default, unless explicitly disabled
3283     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3284         $pciaddr = print_pci_addr("balloon0", $bridges);
3285         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3286     }
3287
3288     if ($conf->{watchdog}) {
3289         my $wdopts = parse_watchdog($conf->{watchdog});
3290         $pciaddr = print_pci_addr("watchdog", $bridges);
3291         my $watchdog = $wdopts->{model} || 'i6300esb';
3292         push @$devices, '-device', "$watchdog$pciaddr";
3293         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3294     }
3295
3296     my $vollist = [];
3297     my $scsicontroller = {};
3298     my $ahcicontroller = {};
3299     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3300
3301     # Add iscsi initiator name if available
3302     if (my $initiator = get_initiator_name()) {
3303         push @$devices, '-iscsi', "initiator-name=$initiator";
3304     }
3305
3306     foreach_drive($conf, sub {
3307         my ($ds, $drive) = @_;
3308
3309         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3310             push @$vollist, $drive->{file};
3311         }
3312
3313         # ignore efidisk here, already added in bios/fw handling code above
3314         return if $drive->{interface} eq 'efidisk';
3315
3316         $use_virtio = 1 if $ds =~ m/^virtio/;
3317
3318         if (drive_is_cdrom ($drive)) {
3319             if ($bootindex_hash->{d}) {
3320                 $drive->{bootindex} = $bootindex_hash->{d};
3321                 $bootindex_hash->{d} += 1;
3322             }
3323         } else {
3324             if ($bootindex_hash->{c}) {
3325                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3326                 $bootindex_hash->{c} += 1;
3327             }
3328         }
3329
3330         if($drive->{interface} eq 'virtio'){
3331            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3332         }
3333
3334         if ($drive->{interface} eq 'scsi') {
3335
3336             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3337
3338             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3339             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3340
3341             my $iothread = '';
3342             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3343                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3344                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3345             } elsif ($drive->{iothread}) {
3346                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3347             }
3348
3349             my $queues = '';
3350             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3351                 $queues = ",num_queues=$drive->{queues}";
3352             }
3353
3354             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3355             $scsicontroller->{$controller}=1;
3356         }
3357
3358         if ($drive->{interface} eq 'sata') {
3359            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3360            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3361            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3362            $ahcicontroller->{$controller}=1;
3363         }
3364
3365         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3366         push @$devices, '-drive',$drive_cmd;
3367         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3368     });
3369
3370     for (my $i = 0; $i < $MAX_NETS; $i++) {
3371          next if !$conf->{"net$i"};
3372          my $d = parse_net($conf->{"net$i"});
3373          next if !$d;
3374
3375          $use_virtio = 1 if $d->{model} eq 'virtio';
3376
3377          if ($bootindex_hash->{n}) {
3378             $d->{bootindex} = $bootindex_hash->{n};
3379             $bootindex_hash->{n} += 1;
3380          }
3381
3382          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3383          push @$devices, '-netdev', $netdevfull;
3384
3385          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3386          push @$devices, '-device', $netdevicefull;
3387     }
3388
3389     if (!$q35) {
3390         # add pci bridges
3391         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3392            $bridges->{1} = 1;
3393            $bridges->{2} = 1;
3394         }
3395
3396         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3397
3398         while (my ($k, $v) = each %$bridges) {
3399             $pciaddr = print_pci_addr("pci.$k");
3400             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3401         }
3402     }
3403
3404     # add custom args
3405     if ($conf->{args}) {
3406         my $aa = PVE::Tools::split_args($conf->{args});
3407         push @$cmd, @$aa;
3408     }
3409
3410     push @$cmd, @$devices;
3411     push @$cmd, '-rtc', join(',', @$rtcFlags)
3412         if scalar(@$rtcFlags);
3413     push @$cmd, '-machine', join(',', @$machineFlags)
3414         if scalar(@$machineFlags);
3415     push @$cmd, '-global', join(',', @$globalFlags)
3416         if scalar(@$globalFlags);
3417
3418     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3419 }
3420
3421 sub vnc_socket {
3422     my ($vmid) = @_;
3423     return "${var_run_tmpdir}/$vmid.vnc";
3424 }
3425
3426 sub spice_port {
3427     my ($vmid) = @_;
3428
3429     my $res = vm_mon_cmd($vmid, 'query-spice');
3430
3431     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3432 }
3433
3434 sub qmp_socket {
3435     my ($vmid, $qga) = @_;
3436     my $sockettype = $qga ? 'qga' : 'qmp';
3437     return "${var_run_tmpdir}/$vmid.$sockettype";
3438 }
3439
3440 sub pidfile_name {
3441     my ($vmid) = @_;
3442     return "${var_run_tmpdir}/$vmid.pid";
3443 }
3444
3445 sub vm_devices_list {
3446     my ($vmid) = @_;
3447
3448     my $res = vm_mon_cmd($vmid, 'query-pci');
3449     my $devices = {};
3450     foreach my $pcibus (@$res) {
3451         foreach my $device (@{$pcibus->{devices}}) {
3452             next if !$device->{'qdev_id'};
3453             if ($device->{'pci_bridge'}) {
3454                 $devices->{$device->{'qdev_id'}} = 1;
3455                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3456                     next if !$bridge_device->{'qdev_id'};
3457                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3458                     $devices->{$device->{'qdev_id'}}++;
3459                 }
3460             } else {
3461                 $devices->{$device->{'qdev_id'}} = 1;
3462             }
3463         }
3464     }
3465
3466     my $resblock = vm_mon_cmd($vmid, 'query-block');
3467     foreach my $block (@$resblock) {
3468         if($block->{device} =~ m/^drive-(\S+)/){
3469                 $devices->{$1} = 1;
3470         }
3471     }
3472
3473     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3474     foreach my $mice (@$resmice) {
3475         if ($mice->{name} eq 'QEMU HID Tablet') {
3476             $devices->{tablet} = 1;
3477             last;
3478         }
3479     }
3480
3481     # for usb devices there is no query-usb
3482     # but we can iterate over the entries in
3483     # qom-list path=/machine/peripheral
3484     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3485     foreach my $per (@$resperipheral) {
3486         if ($per->{name} =~ m/^usb\d+$/) {
3487             $devices->{$per->{name}} = 1;
3488         }
3489     }
3490
3491     return $devices;
3492 }
3493
3494 sub vm_deviceplug {
3495     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3496
3497     my $q35 = machine_type_is_q35($conf);
3498
3499     my $devices_list = vm_devices_list($vmid);
3500     return 1 if defined($devices_list->{$deviceid});
3501
3502     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3503
3504     if ($deviceid eq 'tablet') {
3505
3506         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3507
3508     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3509
3510         die "usb hotplug currently not reliable\n";
3511         # since we can't reliably hot unplug all added usb devices
3512         # and usb passthrough disables live migration
3513         # we disable usb hotplugging for now
3514         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3515
3516     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3517
3518         qemu_iothread_add($vmid, $deviceid, $device);
3519
3520         qemu_driveadd($storecfg, $vmid, $device);
3521         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3522
3523         qemu_deviceadd($vmid, $devicefull);
3524         eval { qemu_deviceaddverify($vmid, $deviceid); };
3525         if (my $err = $@) {
3526             eval { qemu_drivedel($vmid, $deviceid); };
3527             warn $@ if $@;
3528             die $err;
3529         }
3530
3531     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3532
3533
3534         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3535         my $pciaddr = print_pci_addr($deviceid);
3536         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3537
3538         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3539
3540         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3541             qemu_iothread_add($vmid, $deviceid, $device);
3542             $devicefull .= ",iothread=iothread-$deviceid";
3543         }
3544
3545         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3546             $devicefull .= ",num_queues=$device->{queues}";
3547         }
3548
3549         qemu_deviceadd($vmid, $devicefull);
3550         qemu_deviceaddverify($vmid, $deviceid);
3551
3552     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3553
3554         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3555         qemu_driveadd($storecfg, $vmid, $device);
3556
3557         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3558         eval { qemu_deviceadd($vmid, $devicefull); };
3559         if (my $err = $@) {
3560             eval { qemu_drivedel($vmid, $deviceid); };
3561             warn $@ if $@;
3562             die $err;
3563         }
3564
3565     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3566
3567         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3568
3569         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3570         my $use_old_bios_files = undef;
3571         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3572
3573         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3574         qemu_deviceadd($vmid, $netdevicefull);
3575         eval { qemu_deviceaddverify($vmid, $deviceid); };
3576         if (my $err = $@) {
3577             eval { qemu_netdevdel($vmid, $deviceid); };
3578             warn $@ if $@;
3579             die $err;
3580         }
3581
3582     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3583
3584         my $bridgeid = $2;
3585         my $pciaddr = print_pci_addr($deviceid);
3586         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3587
3588         qemu_deviceadd($vmid, $devicefull);
3589         qemu_deviceaddverify($vmid, $deviceid);
3590
3591     } else {
3592         die "can't hotplug device '$deviceid'\n";
3593     }
3594
3595     return 1;
3596 }
3597
3598 # fixme: this should raise exceptions on error!
3599 sub vm_deviceunplug {
3600     my ($vmid, $conf, $deviceid) = @_;
3601
3602     my $devices_list = vm_devices_list($vmid);
3603     return 1 if !defined($devices_list->{$deviceid});
3604
3605     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3606
3607     if ($deviceid eq 'tablet') {
3608
3609         qemu_devicedel($vmid, $deviceid);
3610
3611     } elsif ($deviceid =~ m/^usb\d+$/) {
3612
3613         die "usb hotplug currently not reliable\n";
3614         # when unplugging usb devices this way,
3615         # there may be remaining usb controllers/hubs
3616         # so we disable it for now
3617         qemu_devicedel($vmid, $deviceid);
3618         qemu_devicedelverify($vmid, $deviceid);
3619
3620     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3621
3622         qemu_devicedel($vmid, $deviceid);
3623         qemu_devicedelverify($vmid, $deviceid);
3624         qemu_drivedel($vmid, $deviceid);
3625         qemu_iothread_del($conf, $vmid, $deviceid);
3626
3627     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3628
3629         qemu_devicedel($vmid, $deviceid);
3630         qemu_devicedelverify($vmid, $deviceid);
3631         qemu_iothread_del($conf, $vmid, $deviceid);
3632
3633     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3634
3635         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3636         my $device = parse_drive($deviceid, $conf->{$deviceid});
3637         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3638
3639         qemu_devicedel($vmid, $deviceid);
3640         qemu_drivedel($vmid, $deviceid);
3641         qemu_deletescsihw($conf, $vmid, $deviceid);
3642
3643     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3644
3645         qemu_devicedel($vmid, $deviceid);
3646         qemu_devicedelverify($vmid, $deviceid);
3647         qemu_netdevdel($vmid, $deviceid);
3648
3649     } else {
3650         die "can't unplug device '$deviceid'\n";
3651     }
3652
3653     return 1;
3654 }
3655
3656 sub qemu_deviceadd {
3657     my ($vmid, $devicefull) = @_;
3658
3659     $devicefull = "driver=".$devicefull;
3660     my %options =  split(/[=,]/, $devicefull);
3661
3662     vm_mon_cmd($vmid, "device_add" , %options);
3663 }
3664
3665 sub qemu_devicedel {
3666     my ($vmid, $deviceid) = @_;
3667
3668     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3669 }
3670
3671 sub qemu_iothread_add {
3672     my($vmid, $deviceid, $device) = @_;
3673
3674     if ($device->{iothread}) {
3675         my $iothreads = vm_iothreads_list($vmid);
3676         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3677     }
3678 }
3679
3680 sub qemu_iothread_del {
3681     my($conf, $vmid, $deviceid) = @_;
3682
3683     my $device = parse_drive($deviceid, $conf->{$deviceid});
3684     if ($device->{iothread}) {
3685         my $iothreads = vm_iothreads_list($vmid);
3686         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3687     }
3688 }
3689
3690 sub qemu_objectadd {
3691     my($vmid, $objectid, $qomtype) = @_;
3692
3693     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3694
3695     return 1;
3696 }
3697
3698 sub qemu_objectdel {
3699     my($vmid, $objectid) = @_;
3700
3701     vm_mon_cmd($vmid, "object-del", id => $objectid);
3702
3703     return 1;
3704 }
3705
3706 sub qemu_driveadd {
3707     my ($storecfg, $vmid, $device) = @_;
3708
3709     my $drive = print_drive_full($storecfg, $vmid, $device);
3710     $drive =~ s/\\/\\\\/g;
3711     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3712
3713     # If the command succeeds qemu prints: "OK"
3714     return 1 if $ret =~ m/OK/s;
3715
3716     die "adding drive failed: $ret\n";
3717 }
3718
3719 sub qemu_drivedel {
3720     my($vmid, $deviceid) = @_;
3721
3722     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3723     $ret =~ s/^\s+//;
3724
3725     return 1 if $ret eq "";
3726
3727     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3728     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3729
3730     die "deleting drive $deviceid failed : $ret\n";
3731 }
3732
3733 sub qemu_deviceaddverify {
3734     my ($vmid, $deviceid) = @_;
3735
3736     for (my $i = 0; $i <= 5; $i++) {
3737          my $devices_list = vm_devices_list($vmid);
3738          return 1 if defined($devices_list->{$deviceid});
3739          sleep 1;
3740     }
3741
3742     die "error on hotplug device '$deviceid'\n";
3743 }
3744
3745
3746 sub qemu_devicedelverify {
3747     my ($vmid, $deviceid) = @_;
3748
3749     # need to verify that the device is correctly removed as device_del
3750     # is async and empty return is not reliable
3751
3752     for (my $i = 0; $i <= 5; $i++) {
3753          my $devices_list = vm_devices_list($vmid);
3754          return 1 if !defined($devices_list->{$deviceid});
3755          sleep 1;
3756     }
3757
3758     die "error on hot-unplugging device '$deviceid'\n";
3759 }
3760
3761 sub qemu_findorcreatescsihw {
3762     my ($storecfg, $conf, $vmid, $device) = @_;
3763
3764     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3765
3766     my $scsihwid="$controller_prefix$controller";
3767     my $devices_list = vm_devices_list($vmid);
3768
3769     if(!defined($devices_list->{$scsihwid})) {
3770         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3771     }
3772
3773     return 1;
3774 }
3775
3776 sub qemu_deletescsihw {
3777     my ($conf, $vmid, $opt) = @_;
3778
3779     my $device = parse_drive($opt, $conf->{$opt});
3780
3781     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3782         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3783         return 1;
3784     }
3785
3786     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3787
3788     my $devices_list = vm_devices_list($vmid);
3789     foreach my $opt (keys %{$devices_list}) {
3790         if (PVE::QemuServer::is_valid_drivename($opt)) {
3791             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3792             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3793                 return 1;
3794             }
3795         }
3796     }
3797
3798     my $scsihwid="scsihw$controller";
3799
3800     vm_deviceunplug($vmid, $conf, $scsihwid);
3801
3802     return 1;
3803 }
3804
3805 sub qemu_add_pci_bridge {
3806     my ($storecfg, $conf, $vmid, $device) = @_;
3807
3808     my $bridges = {};
3809
3810     my $bridgeid;
3811
3812     print_pci_addr($device, $bridges);
3813
3814     while (my ($k, $v) = each %$bridges) {
3815         $bridgeid = $k;
3816     }
3817     return 1 if !defined($bridgeid) || $bridgeid < 1;
3818
3819     my $bridge = "pci.$bridgeid";
3820     my $devices_list = vm_devices_list($vmid);
3821
3822     if (!defined($devices_list->{$bridge})) {
3823         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3824     }
3825
3826     return 1;
3827 }
3828
3829 sub qemu_set_link_status {
3830     my ($vmid, $device, $up) = @_;
3831
3832     vm_mon_cmd($vmid, "set_link", name => $device,
3833                up => $up ? JSON::true : JSON::false);
3834 }
3835
3836 sub qemu_netdevadd {
3837     my ($vmid, $conf, $device, $deviceid) = @_;
3838
3839     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3840     my %options =  split(/[=,]/, $netdev);
3841
3842     vm_mon_cmd($vmid, "netdev_add",  %options);
3843     return 1;
3844 }
3845
3846 sub qemu_netdevdel {
3847     my ($vmid, $deviceid) = @_;
3848
3849     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3850 }
3851
3852 sub qemu_usb_hotplug {
3853     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3854
3855     return if !$device;
3856
3857     # remove the old one first
3858     vm_deviceunplug($vmid, $conf, $deviceid);
3859
3860     # check if xhci controller is necessary and available
3861     if ($device->{usb3}) {
3862
3863         my $devicelist = vm_devices_list($vmid);
3864
3865         if (!$devicelist->{xhci}) {
3866             my $pciaddr = print_pci_addr("xhci");
3867             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3868         }
3869     }
3870     my $d = parse_usb_device($device->{host});
3871     $d->{usb3} = $device->{usb3};
3872
3873     # add the new one
3874     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3875 }
3876
3877 sub qemu_cpu_hotplug {
3878     my ($vmid, $conf, $vcpus) = @_;
3879
3880     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3881
3882     my $sockets = 1;
3883     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3884     $sockets = $conf->{sockets} if  $conf->{sockets};
3885     my $cores = $conf->{cores} || 1;
3886     my $maxcpus = $sockets * $cores;
3887
3888     $vcpus = $maxcpus if !$vcpus;
3889
3890     die "you can't add more vcpus than maxcpus\n"
3891         if $vcpus > $maxcpus;
3892
3893     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3894
3895     if ($vcpus < $currentvcpus) {
3896
3897         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3898
3899             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3900                 qemu_devicedel($vmid, "cpu$i");
3901                 my $retry = 0;
3902                 my $currentrunningvcpus = undef;
3903                 while (1) {
3904                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3905                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3906                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3907                     $retry++;
3908                     sleep 1;
3909                 }
3910                 #update conf after each succesfull cpu unplug
3911                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3912                 PVE::QemuConfig->write_config($vmid, $conf);
3913             }
3914         } else {
3915             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3916         }
3917
3918         return;
3919     }
3920
3921     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3922     die "vcpus in running vm does not match its configuration\n"
3923         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3924
3925     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3926
3927         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3928             my $cpustr = print_cpu_device($conf, $i);
3929             qemu_deviceadd($vmid, $cpustr);
3930
3931             my $retry = 0;
3932             my $currentrunningvcpus = undef;
3933             while (1) {
3934                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3935                 last if scalar(@{$currentrunningvcpus}) == $i;
3936                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3937                 sleep 1;
3938                 $retry++;
3939             }
3940             #update conf after each succesfull cpu hotplug
3941             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3942             PVE::QemuConfig->write_config($vmid, $conf);
3943         }
3944     } else {
3945
3946         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3947             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3948         }
3949     }
3950 }
3951
3952 sub qemu_block_set_io_throttle {
3953     my ($vmid, $deviceid,
3954         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3955         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3956         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3957         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3958
3959     return if !check_running($vmid) ;
3960
3961     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3962         bps => int($bps),
3963         bps_rd => int($bps_rd),
3964         bps_wr => int($bps_wr),
3965         iops => int($iops),
3966         iops_rd => int($iops_rd),
3967         iops_wr => int($iops_wr),
3968         bps_max => int($bps_max),
3969         bps_rd_max => int($bps_rd_max),
3970         bps_wr_max => int($bps_wr_max),
3971         iops_max => int($iops_max),
3972         iops_rd_max => int($iops_rd_max),
3973         iops_wr_max => int($iops_wr_max),
3974         bps_max_length => int($bps_max_length),
3975         bps_rd_max_length => int($bps_rd_max_length),
3976         bps_wr_max_length => int($bps_wr_max_length),
3977         iops_max_length => int($iops_max_length),
3978         iops_rd_max_length => int($iops_rd_max_length),
3979         iops_wr_max_length => int($iops_wr_max_length),
3980     );
3981
3982 }
3983
3984 # old code, only used to shutdown old VM after update
3985 sub __read_avail {
3986     my ($fh, $timeout) = @_;
3987
3988     my $sel = new IO::Select;
3989     $sel->add($fh);
3990
3991     my $res = '';
3992     my $buf;
3993
3994     my @ready;
3995     while (scalar (@ready = $sel->can_read($timeout))) {
3996         my $count;
3997         if ($count = $fh->sysread($buf, 8192)) {
3998             if ($buf =~ /^(.*)\(qemu\) $/s) {
3999                 $res .= $1;
4000                 last;
4001             } else {
4002                 $res .= $buf;
4003             }
4004         } else {
4005             if (!defined($count)) {
4006                 die "$!\n";
4007             }
4008             last;
4009         }
4010     }
4011
4012     die "monitor read timeout\n" if !scalar(@ready);
4013
4014     return $res;
4015 }
4016
4017 # old code, only used to shutdown old VM after update
4018 sub vm_monitor_command {
4019     my ($vmid, $cmdstr, $nocheck) = @_;
4020
4021     my $res;
4022
4023     eval {
4024         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4025
4026         my $sname = "${var_run_tmpdir}/$vmid.mon";
4027
4028         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
4029             die "unable to connect to VM $vmid socket - $!\n";
4030
4031         my $timeout = 3;
4032
4033         # hack: migrate sometime blocks the monitor (when migrate_downtime
4034         # is set)
4035         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4036             $timeout = 60*60; # 1 hour
4037         }
4038
4039         # read banner;
4040         my $data = __read_avail($sock, $timeout);
4041
4042         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
4043             die "got unexpected qemu monitor banner\n";
4044         }
4045
4046         my $sel = new IO::Select;
4047         $sel->add($sock);
4048
4049         if (!scalar(my @ready = $sel->can_write($timeout))) {
4050             die "monitor write error - timeout";
4051         }
4052
4053         my $fullcmd = "$cmdstr\r";
4054
4055         # syslog('info', "VM $vmid monitor command: $cmdstr");
4056
4057         my $b;
4058         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
4059             die "monitor write error - $!";
4060         }
4061
4062         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4063
4064         $timeout = 20;
4065
4066         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4067             $timeout = 60*60; # 1 hour
4068         } elsif ($cmdstr =~ m/^(eject|change)/) {
4069             $timeout = 60; # note: cdrom mount command is slow
4070         }
4071         if ($res = __read_avail($sock, $timeout)) {
4072
4073             my @lines = split("\r?\n", $res);
4074
4075             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4076
4077             $res = join("\n", @lines);
4078             $res .= "\n";
4079         }
4080     };
4081
4082     my $err = $@;
4083
4084     if ($err) {
4085         syslog("err", "VM $vmid monitor command failed - $err");
4086         die $err;
4087     }
4088
4089     return $res;
4090 }
4091
4092 sub qemu_block_resize {
4093     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4094
4095     my $running = check_running($vmid);
4096
4097     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4098
4099     return if !$running;
4100
4101     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4102
4103 }
4104
4105 sub qemu_volume_snapshot {
4106     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4107
4108     my $running = check_running($vmid);
4109
4110     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4111         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4112     } else {
4113         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4114     }
4115 }
4116
4117 sub qemu_volume_snapshot_delete {
4118     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4119
4120     my $running = check_running($vmid);
4121
4122     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4123         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4124     } else {
4125         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4126     }
4127 }
4128
4129 sub set_migration_caps {
4130     my ($vmid) = @_;
4131
4132     my $cap_ref = [];
4133
4134     my $enabled_cap = {
4135         "auto-converge" => 1,
4136         "xbzrle" => 1,
4137         "x-rdma-pin-all" => 0,
4138         "zero-blocks" => 0,
4139         "compress" => 0
4140     };
4141
4142     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4143
4144     for my $supported_capability (@$supported_capabilities) {
4145         push @$cap_ref, {
4146             capability => $supported_capability->{capability},
4147             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4148         };
4149     }
4150
4151     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4152 }
4153
4154 my $fast_plug_option = {
4155     'lock' => 1,
4156     'name' => 1,
4157     'onboot' => 1,
4158     'shares' => 1,
4159     'startup' => 1,
4160     'description' => 1,
4161     'protection' => 1,
4162     'vmstatestorage' => 1,
4163 };
4164
4165 # hotplug changes in [PENDING]
4166 # $selection hash can be used to only apply specified options, for
4167 # example: { cores => 1 } (only apply changed 'cores')
4168 # $errors ref is used to return error messages
4169 sub vmconfig_hotplug_pending {
4170     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4171
4172     my $defaults = load_defaults();
4173
4174     # commit values which do not have any impact on running VM first
4175     # Note: those option cannot raise errors, we we do not care about
4176     # $selection and always apply them.
4177
4178     my $add_error = sub {
4179         my ($opt, $msg) = @_;
4180         $errors->{$opt} = "hotplug problem - $msg";
4181     };
4182
4183     my $changes = 0;
4184     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4185         if ($fast_plug_option->{$opt}) {
4186             $conf->{$opt} = $conf->{pending}->{$opt};
4187             delete $conf->{pending}->{$opt};
4188             $changes = 1;
4189         }
4190     }
4191
4192     if ($changes) {
4193         PVE::QemuConfig->write_config($vmid, $conf);
4194         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4195     }
4196
4197     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4198
4199     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4200     while (my ($opt, $force) = each %$pending_delete_hash) {
4201         next if $selection && !$selection->{$opt};
4202         eval {
4203             if ($opt eq 'hotplug') {
4204                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4205             } elsif ($opt eq 'tablet') {
4206                 die "skip\n" if !$hotplug_features->{usb};
4207                 if ($defaults->{tablet}) {
4208                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4209                 } else {
4210                     vm_deviceunplug($vmid, $conf, $opt);
4211                 }
4212             } elsif ($opt =~ m/^usb\d+/) {
4213                 die "skip\n";
4214                 # since we cannot reliably hot unplug usb devices
4215                 # we are disabling it
4216                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4217                 vm_deviceunplug($vmid, $conf, $opt);
4218             } elsif ($opt eq 'vcpus') {
4219                 die "skip\n" if !$hotplug_features->{cpu};
4220                 qemu_cpu_hotplug($vmid, $conf, undef);
4221             } elsif ($opt eq 'balloon') {
4222                 # enable balloon device is not hotpluggable
4223                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4224             } elsif ($fast_plug_option->{$opt}) {
4225                 # do nothing
4226             } elsif ($opt =~ m/^net(\d+)$/) {
4227                 die "skip\n" if !$hotplug_features->{network};
4228                 vm_deviceunplug($vmid, $conf, $opt);
4229             } elsif (is_valid_drivename($opt)) {
4230                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4231                 vm_deviceunplug($vmid, $conf, $opt);
4232                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4233             } elsif ($opt =~ m/^memory$/) {
4234                 die "skip\n" if !$hotplug_features->{memory};
4235                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4236             } elsif ($opt eq 'cpuunits') {
4237                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4238             } elsif ($opt eq 'cpulimit') {
4239                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4240             } else {
4241                 die "skip\n";
4242             }
4243         };
4244         if (my $err = $@) {
4245             &$add_error($opt, $err) if $err ne "skip\n";
4246         } else {
4247             # save new config if hotplug was successful
4248             delete $conf->{$opt};
4249             vmconfig_undelete_pending_option($conf, $opt);
4250             PVE::QemuConfig->write_config($vmid, $conf);
4251             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4252         }
4253     }
4254
4255     foreach my $opt (keys %{$conf->{pending}}) {
4256         next if $selection && !$selection->{$opt};
4257         my $value = $conf->{pending}->{$opt};
4258         eval {
4259             if ($opt eq 'hotplug') {
4260                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4261             } elsif ($opt eq 'tablet') {
4262                 die "skip\n" if !$hotplug_features->{usb};
4263                 if ($value == 1) {
4264                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4265                 } elsif ($value == 0) {
4266                     vm_deviceunplug($vmid, $conf, $opt);
4267                 }
4268             } elsif ($opt =~ m/^usb\d+$/) {
4269                 die "skip\n";
4270                 # since we cannot reliably hot unplug usb devices
4271                 # we are disabling it
4272                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4273                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4274                 die "skip\n" if !$d;
4275                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4276             } elsif ($opt eq 'vcpus') {
4277                 die "skip\n" if !$hotplug_features->{cpu};
4278                 qemu_cpu_hotplug($vmid, $conf, $value);
4279             } elsif ($opt eq 'balloon') {
4280                 # enable/disable balloning device is not hotpluggable
4281                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4282                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4283                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4284
4285                 # allow manual ballooning if shares is set to zero
4286                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4287                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4288                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4289                 }
4290             } elsif ($opt =~ m/^net(\d+)$/) {
4291                 # some changes can be done without hotplug
4292                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4293                                     $vmid, $opt, $value);
4294             } elsif (is_valid_drivename($opt)) {
4295                 # some changes can be done without hotplug
4296                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4297                                      $vmid, $opt, $value, 1);
4298             } elsif ($opt =~ m/^memory$/) { #dimms
4299                 die "skip\n" if !$hotplug_features->{memory};
4300                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4301             } elsif ($opt eq 'cpuunits') {
4302                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4303             } elsif ($opt eq 'cpulimit') {
4304                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4305                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4306             } else {
4307                 die "skip\n";  # skip non-hot-pluggable options
4308             }
4309         };
4310         if (my $err = $@) {
4311             &$add_error($opt, $err) if $err ne "skip\n";
4312         } else {
4313             # save new config if hotplug was successful
4314             $conf->{$opt} = $value;
4315             delete $conf->{pending}->{$opt};
4316             PVE::QemuConfig->write_config($vmid, $conf);
4317             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4318         }
4319     }
4320 }
4321
4322 sub try_deallocate_drive {
4323     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4324
4325     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4326         my $volid = $drive->{file};
4327         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4328             my $sid = PVE::Storage::parse_volume_id($volid);
4329             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4330
4331             # check if the disk is really unused
4332             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4333                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4334             PVE::Storage::vdisk_free($storecfg, $volid);
4335             return 1;
4336         } else {
4337             # If vm is not owner of this disk remove from config
4338             return 1;
4339         }
4340     }
4341
4342     return undef;
4343 }
4344
4345 sub vmconfig_delete_or_detach_drive {
4346     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4347
4348     my $drive = parse_drive($opt, $conf->{$opt});
4349
4350     my $rpcenv = PVE::RPCEnvironment::get();
4351     my $authuser = $rpcenv->get_user();
4352
4353     if ($force) {
4354         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4355         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4356     } else {
4357         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4358     }
4359 }
4360
4361 sub vmconfig_apply_pending {
4362     my ($vmid, $conf, $storecfg) = @_;
4363
4364     # cold plug
4365
4366     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4367     while (my ($opt, $force) = each %$pending_delete_hash) {
4368         die "internal error" if $opt =~ m/^unused/;
4369         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4370         if (!defined($conf->{$opt})) {
4371             vmconfig_undelete_pending_option($conf, $opt);
4372             PVE::QemuConfig->write_config($vmid, $conf);
4373         } elsif (is_valid_drivename($opt)) {
4374             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4375             vmconfig_undelete_pending_option($conf, $opt);
4376             delete $conf->{$opt};
4377             PVE::QemuConfig->write_config($vmid, $conf);
4378         } else {
4379             vmconfig_undelete_pending_option($conf, $opt);
4380             delete $conf->{$opt};
4381             PVE::QemuConfig->write_config($vmid, $conf);
4382         }
4383     }
4384
4385     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4386
4387     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4388         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4389
4390         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4391             # skip if nothing changed
4392         } elsif (is_valid_drivename($opt)) {
4393             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4394                 if defined($conf->{$opt});
4395             $conf->{$opt} = $conf->{pending}->{$opt};
4396         } else {
4397             $conf->{$opt} = $conf->{pending}->{$opt};
4398         }
4399
4400         delete $conf->{pending}->{$opt};
4401         PVE::QemuConfig->write_config($vmid, $conf);
4402     }
4403 }
4404
4405 my $safe_num_ne = sub {
4406     my ($a, $b) = @_;
4407
4408     return 0 if !defined($a) && !defined($b);
4409     return 1 if !defined($a);
4410     return 1 if !defined($b);
4411
4412     return $a != $b;
4413 };
4414
4415 my $safe_string_ne = sub {
4416     my ($a, $b) = @_;
4417
4418     return 0 if !defined($a) && !defined($b);
4419     return 1 if !defined($a);
4420     return 1 if !defined($b);
4421
4422     return $a ne $b;
4423 };
4424
4425 sub vmconfig_update_net {
4426     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4427
4428     my $newnet = parse_net($value);
4429
4430     if ($conf->{$opt}) {
4431         my $oldnet = parse_net($conf->{$opt});
4432
4433         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4434             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4435             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4436             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4437
4438             # for non online change, we try to hot-unplug
4439             die "skip\n" if !$hotplug;
4440             vm_deviceunplug($vmid, $conf, $opt);
4441         } else {
4442
4443             die "internal error" if $opt !~ m/net(\d+)/;
4444             my $iface = "tap${vmid}i$1";
4445
4446             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4447                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4448                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4449                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4450                 PVE::Network::tap_unplug($iface);
4451                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4452             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4453                 # Rate can be applied on its own but any change above needs to
4454                 # include the rate in tap_plug since OVS resets everything.
4455                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4456             }
4457
4458             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4459                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4460             }
4461
4462             return 1;
4463         }
4464     }
4465
4466     if ($hotplug) {
4467         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4468     } else {
4469         die "skip\n";
4470     }
4471 }
4472
4473 sub vmconfig_update_disk {
4474     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4475
4476     # fixme: do we need force?
4477
4478     my $drive = parse_drive($opt, $value);
4479
4480     if ($conf->{$opt}) {
4481
4482         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4483
4484             my $media = $drive->{media} || 'disk';
4485             my $oldmedia = $old_drive->{media} || 'disk';
4486             die "unable to change media type\n" if $media ne $oldmedia;
4487
4488             if (!drive_is_cdrom($old_drive)) {
4489
4490                 if ($drive->{file} ne $old_drive->{file}) {
4491
4492                     die "skip\n" if !$hotplug;
4493
4494                     # unplug and register as unused
4495                     vm_deviceunplug($vmid, $conf, $opt);
4496                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4497
4498                 } else {
4499                     # update existing disk
4500
4501                     # skip non hotpluggable value
4502                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4503                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4504                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4505                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4506                         die "skip\n";
4507                     }
4508
4509                     # apply throttle
4510                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4511                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4512                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4513                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4514                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4515                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4516                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4517                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4518                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4519                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4520                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4521                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4522                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4523                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4524                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4525                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4526                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4527                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4528
4529                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4530                                                    ($drive->{mbps} || 0)*1024*1024,
4531                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4532                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4533                                                    $drive->{iops} || 0,
4534                                                    $drive->{iops_rd} || 0,
4535                                                    $drive->{iops_wr} || 0,
4536                                                    ($drive->{mbps_max} || 0)*1024*1024,
4537                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4538                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4539                                                    $drive->{iops_max} || 0,
4540                                                    $drive->{iops_rd_max} || 0,
4541                                                    $drive->{iops_wr_max} || 0,
4542                                                    $drive->{bps_max_length} || 1,
4543                                                    $drive->{bps_rd_max_length} || 1,
4544                                                    $drive->{bps_wr_max_length} || 1,
4545                                                    $drive->{iops_max_length} || 1,
4546                                                    $drive->{iops_rd_max_length} || 1,
4547                                                    $drive->{iops_wr_max_length} || 1);
4548
4549                     }
4550
4551                     return 1;
4552                 }
4553
4554             } else { # cdrom
4555
4556                 if ($drive->{file} eq 'none') {
4557                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4558                 } else {
4559                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4560                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4561                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4562                 }
4563
4564                 return 1;
4565             }
4566         }
4567     }
4568
4569     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4570     # hotplug new disks
4571     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4572     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4573 }
4574
4575 sub vm_start {
4576     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4577         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4578
4579     PVE::QemuConfig->lock_config($vmid, sub {
4580         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4581
4582         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4583
4584         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4585
4586         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4587
4588         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4589             vmconfig_apply_pending($vmid, $conf, $storecfg);
4590             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4591         }
4592
4593         my $defaults = load_defaults();
4594
4595         # set environment variable useful inside network script
4596         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4597
4598         my $local_volumes = {};
4599
4600         if ($targetstorage) {
4601             foreach_drive($conf, sub {
4602                 my ($ds, $drive) = @_;
4603
4604                 return if drive_is_cdrom($drive);
4605
4606                 my $volid = $drive->{file};
4607
4608                 return if !$volid;
4609
4610                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4611
4612                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4613                 return if $scfg->{shared};
4614                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4615             });
4616
4617             my $format = undef;
4618
4619             foreach my $opt (sort keys %$local_volumes) {
4620
4621                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4622                 my $drive = parse_drive($opt, $conf->{$opt});
4623
4624                 #if remote storage is specified, use default format
4625                 if ($targetstorage && $targetstorage ne "1") {
4626                     $storeid = $targetstorage;
4627                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4628                     $format = $defFormat;
4629                 } else {
4630                     #else we use same format than original
4631                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4632                     $format = qemu_img_format($scfg, $volid);
4633                 }
4634
4635                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4636                 my $newdrive = $drive;
4637                 $newdrive->{format} = $format;
4638                 $newdrive->{file} = $newvolid;
4639                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4640                 $local_volumes->{$opt} = $drivestr;
4641                 #pass drive to conf for command line
4642                 $conf->{$opt} = $drivestr;
4643             }
4644         }
4645
4646         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4647
4648         my $migrate_port = 0;
4649         my $migrate_uri;
4650         if ($statefile) {
4651             if ($statefile eq 'tcp') {
4652                 my $localip = "localhost";
4653                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4654                 my $nodename = PVE::INotify::nodename();
4655
4656                 if (!defined($migration_type)) {
4657                     if (defined($datacenterconf->{migration}->{type})) {
4658                         $migration_type = $datacenterconf->{migration}->{type};
4659                     } else {
4660                         $migration_type = 'secure';
4661                     }
4662                 }
4663
4664                 if ($migration_type eq 'insecure') {
4665                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4666                     if ($migrate_network_addr) {
4667                         $localip = $migrate_network_addr;
4668                     } else {
4669                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4670                     }
4671
4672                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4673                 }
4674
4675                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4676                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4677                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4678                 push @$cmd, '-incoming', $migrate_uri;
4679                 push @$cmd, '-S';
4680
4681             } elsif ($statefile eq 'unix') {
4682                 # should be default for secure migrations as a ssh TCP forward
4683                 # tunnel is not deterministic reliable ready and fails regurarly
4684                 # to set up in time, so use UNIX socket forwards
4685                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4686                 unlink $socket_addr;
4687
4688                 $migrate_uri = "unix:$socket_addr";
4689
4690                 push @$cmd, '-incoming', $migrate_uri;
4691                 push @$cmd, '-S';
4692
4693             } else {
4694                 push @$cmd, '-loadstate', $statefile;
4695             }
4696         } elsif ($paused) {
4697             push @$cmd, '-S';
4698         }
4699
4700         # host pci devices
4701         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4702           my $d = parse_hostpci($conf->{"hostpci$i"});
4703           next if !$d;
4704           my $pcidevices = $d->{pciid};
4705           foreach my $pcidevice (@$pcidevices) {
4706                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4707
4708                 my $info = pci_device_info("0000:$pciid");
4709                 die "IOMMU not present\n" if !check_iommu_support();
4710                 die "no pci device info for device '$pciid'\n" if !$info;
4711                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4712                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4713           }
4714         }
4715
4716         PVE::Storage::activate_volumes($storecfg, $vollist);
4717
4718         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4719             my $cmd = [];
4720             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4721             eval  { run_command($cmd); };
4722         }
4723
4724         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4725                                                   : $defaults->{cpuunits};
4726
4727         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4728         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4729
4730         my %properties = (
4731             Slice => 'qemu.slice',
4732             KillMode => 'none',
4733             CPUShares => $cpuunits
4734         );
4735
4736         if (my $cpulimit = $conf->{cpulimit}) {
4737             $properties{CPUQuota} = int($cpulimit * 100);
4738         }
4739         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4740
4741         if ($conf->{hugepages}) {
4742
4743             my $code = sub {
4744                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4745                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4746
4747                 PVE::QemuServer::Memory::hugepages_mount();
4748                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4749
4750                 eval  {
4751                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4752                     run_command($cmd, %run_params);
4753                 };
4754
4755                 if (my $err = $@) {
4756                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4757                     die $err;
4758                 }
4759
4760                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4761             };
4762             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4763
4764         } else {
4765             eval  {
4766                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4767                 run_command($cmd, %run_params);
4768             };
4769         }
4770
4771         if (my $err = $@) {
4772             # deactivate volumes if start fails
4773             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4774             die "start failed: $err";
4775         }
4776
4777         print "migration listens on $migrate_uri\n" if $migrate_uri;
4778
4779         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4780             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4781             warn $@ if $@;
4782         }
4783
4784         #start nbd server for storage migration
4785         if ($targetstorage) {
4786             my $nodename = PVE::INotify::nodename();
4787             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4788             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4789             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4790             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4791
4792             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4793
4794             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4795
4796             foreach my $opt (sort keys %$local_volumes) {
4797                 my $volid = $local_volumes->{$opt};
4798                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4799                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4800                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4801             }
4802         }
4803
4804         if ($migratedfrom) {
4805             eval {
4806                 set_migration_caps($vmid);
4807             };
4808             warn $@ if $@;
4809
4810             if ($spice_port) {
4811                 print "spice listens on port $spice_port\n";
4812                 if ($spice_ticket) {
4813                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4814                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4815                 }
4816             }
4817
4818         } else {
4819             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4820                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4821                     if $conf->{balloon};
4822             }
4823
4824             foreach my $opt (keys %$conf) {
4825                 next if $opt !~  m/^net\d+$/;
4826                 my $nicconf = parse_net($conf->{$opt});
4827                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4828             }
4829         }
4830
4831         vm_mon_cmd_nocheck($vmid, 'qom-set',
4832                     path => "machine/peripheral/balloon0",
4833                     property => "guest-stats-polling-interval",
4834                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4835
4836     });
4837 }
4838
4839 sub vm_mon_cmd {
4840     my ($vmid, $execute, %params) = @_;
4841
4842     my $cmd = { execute => $execute, arguments => \%params };
4843     vm_qmp_command($vmid, $cmd);
4844 }
4845
4846 sub vm_mon_cmd_nocheck {
4847     my ($vmid, $execute, %params) = @_;
4848
4849     my $cmd = { execute => $execute, arguments => \%params };
4850     vm_qmp_command($vmid, $cmd, 1);
4851 }
4852
4853 sub vm_qmp_command {
4854     my ($vmid, $cmd, $nocheck) = @_;
4855
4856     my $res;
4857
4858     my $timeout;
4859     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4860         $timeout = $cmd->{arguments}->{timeout};
4861         delete $cmd->{arguments}->{timeout};
4862     }
4863
4864     eval {
4865         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4866         my $sname = qmp_socket($vmid);
4867         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4868             my $qmpclient = PVE::QMPClient->new();
4869
4870             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4871         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4872             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4873                 if scalar(%{$cmd->{arguments}});
4874             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4875         } else {
4876             die "unable to open monitor socket\n";
4877         }
4878     };
4879     if (my $err = $@) {
4880         syslog("err", "VM $vmid qmp command failed - $err");
4881         die $err;
4882     }
4883
4884     return $res;
4885 }
4886
4887 sub vm_human_monitor_command {
4888     my ($vmid, $cmdline) = @_;
4889
4890     my $res;
4891
4892     my $cmd = {
4893         execute => 'human-monitor-command',
4894         arguments => { 'command-line' => $cmdline},
4895     };
4896
4897     return vm_qmp_command($vmid, $cmd);
4898 }
4899
4900 sub vm_commandline {
4901     my ($storecfg, $vmid) = @_;
4902
4903     my $conf = PVE::QemuConfig->load_config($vmid);
4904
4905     my $defaults = load_defaults();
4906
4907     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4908
4909     return PVE::Tools::cmd2string($cmd);
4910 }
4911
4912 sub vm_reset {
4913     my ($vmid, $skiplock) = @_;
4914
4915     PVE::QemuConfig->lock_config($vmid, sub {
4916
4917         my $conf = PVE::QemuConfig->load_config($vmid);
4918
4919         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4920
4921         vm_mon_cmd($vmid, "system_reset");
4922     });
4923 }
4924
4925 sub get_vm_volumes {
4926     my ($conf) = @_;
4927
4928     my $vollist = [];
4929     foreach_volid($conf, sub {
4930         my ($volid, $attr) = @_;
4931
4932         return if $volid =~ m|^/|;
4933
4934         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4935         return if !$sid;
4936
4937         push @$vollist, $volid;
4938     });
4939
4940     return $vollist;
4941 }
4942
4943 sub vm_stop_cleanup {
4944     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4945
4946     eval {
4947
4948         if (!$keepActive) {
4949             my $vollist = get_vm_volumes($conf);
4950             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4951         }
4952
4953         foreach my $ext (qw(mon qmp pid vnc qga)) {
4954             unlink "/var/run/qemu-server/${vmid}.$ext";
4955         }
4956
4957         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4958     };
4959     warn $@ if $@; # avoid errors - just warn
4960 }
4961
4962 # Note: use $nockeck to skip tests if VM configuration file exists.
4963 # We need that when migration VMs to other nodes (files already moved)
4964 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4965 sub vm_stop {
4966     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4967
4968     $force = 1 if !defined($force) && !$shutdown;
4969
4970     if ($migratedfrom){
4971         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4972         kill 15, $pid if $pid;
4973         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4974         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4975         return;
4976     }
4977
4978     PVE::QemuConfig->lock_config($vmid, sub {
4979
4980         my $pid = check_running($vmid, $nocheck);
4981         return if !$pid;
4982
4983         my $conf;
4984         if (!$nocheck) {
4985             $conf = PVE::QemuConfig->load_config($vmid);
4986             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4987             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4988                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4989                 $timeout = $opts->{down} if $opts->{down};
4990             }
4991         }
4992
4993         $timeout = 60 if !defined($timeout);
4994
4995         eval {
4996             if ($shutdown) {
4997                 if (defined($conf) && $conf->{agent}) {
4998                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4999                 } else {
5000                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5001                 }
5002             } else {
5003                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5004             }
5005         };
5006         my $err = $@;
5007
5008         if (!$err) {
5009             my $count = 0;
5010             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5011                 $count++;
5012                 sleep 1;
5013             }
5014
5015             if ($count >= $timeout) {
5016                 if ($force) {
5017                     warn "VM still running - terminating now with SIGTERM\n";
5018                     kill 15, $pid;
5019                 } else {
5020                     die "VM quit/powerdown failed - got timeout\n";
5021                 }
5022             } else {
5023                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5024                 return;
5025             }
5026         } else {
5027             if ($force) {
5028                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5029                 kill 15, $pid;
5030             } else {
5031                 die "VM quit/powerdown failed\n";
5032             }
5033         }
5034
5035         # wait again
5036         $timeout = 10;
5037
5038         my $count = 0;
5039         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5040             $count++;
5041             sleep 1;
5042         }
5043
5044         if ($count >= $timeout) {
5045             warn "VM still running - terminating now with SIGKILL\n";
5046             kill 9, $pid;
5047             sleep 1;
5048         }
5049
5050         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5051    });
5052 }
5053
5054 sub vm_suspend {
5055     my ($vmid, $skiplock) = @_;
5056
5057     PVE::QemuConfig->lock_config($vmid, sub {
5058
5059         my $conf = PVE::QemuConfig->load_config($vmid);
5060
5061         PVE::QemuConfig->check_lock($conf)
5062             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5063
5064         vm_mon_cmd($vmid, "stop");
5065     });
5066 }
5067
5068 sub vm_resume {
5069     my ($vmid, $skiplock, $nocheck) = @_;
5070
5071     PVE::QemuConfig->lock_config($vmid, sub {
5072
5073         if (!$nocheck) {
5074
5075             my $conf = PVE::QemuConfig->load_config($vmid);
5076
5077             PVE::QemuConfig->check_lock($conf)
5078                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5079
5080             vm_mon_cmd($vmid, "cont");
5081
5082         } else {
5083             vm_mon_cmd_nocheck($vmid, "cont");
5084         }
5085     });
5086 }
5087
5088 sub vm_sendkey {
5089     my ($vmid, $skiplock, $key) = @_;
5090
5091     PVE::QemuConfig->lock_config($vmid, sub {
5092
5093         my $conf = PVE::QemuConfig->load_config($vmid);
5094
5095         # there is no qmp command, so we use the human monitor command
5096         vm_human_monitor_command($vmid, "sendkey $key");
5097     });
5098 }
5099
5100 sub vm_destroy {
5101     my ($storecfg, $vmid, $skiplock) = @_;
5102
5103     PVE::QemuConfig->lock_config($vmid, sub {
5104
5105         my $conf = PVE::QemuConfig->load_config($vmid);
5106
5107         if (!check_running($vmid)) {
5108             destroy_vm($storecfg, $vmid, undef, $skiplock);
5109         } else {
5110             die "VM $vmid is running - destroy failed\n";
5111         }
5112     });
5113 }
5114
5115 # pci helpers
5116
5117 sub file_write {
5118     my ($filename, $buf) = @_;
5119
5120     my $fh = IO::File->new($filename, "w");
5121     return undef if !$fh;
5122
5123     my $res = print $fh $buf;
5124
5125     $fh->close();
5126
5127     return $res;
5128 }
5129
5130 sub pci_device_info {
5131     my ($name) = @_;
5132
5133     my $res;
5134
5135     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5136     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5137
5138     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5139     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5140
5141     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5142     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5143
5144     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5145     return undef if !defined($product) || $product !~ s/^0x//;
5146
5147     $res = {
5148         name => $name,
5149         vendor => $vendor,
5150         product => $product,
5151         domain => $domain,
5152         bus => $bus,
5153         slot => $slot,
5154         func => $func,
5155         irq => $irq,
5156         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5157     };
5158
5159     return $res;
5160 }
5161
5162 sub pci_dev_reset {
5163     my ($dev) = @_;
5164
5165     my $name = $dev->{name};
5166
5167     my $fn = "$pcisysfs/devices/$name/reset";
5168
5169     return file_write($fn, "1");
5170 }
5171
5172 sub pci_dev_bind_to_vfio {
5173     my ($dev) = @_;
5174
5175     my $name = $dev->{name};
5176
5177     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5178
5179     if (!-d $vfio_basedir) {
5180         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5181     }
5182     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5183
5184     my $testdir = "$vfio_basedir/$name";
5185     return 1 if -d $testdir;
5186
5187     my $data = "$dev->{vendor} $dev->{product}";
5188     return undef if !file_write("$vfio_basedir/new_id", $data);
5189
5190     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5191     if (!file_write($fn, $name)) {
5192         return undef if -f $fn;
5193     }
5194
5195     $fn = "$vfio_basedir/bind";
5196     if (! -d $testdir) {
5197         return undef if !file_write($fn, $name);
5198     }
5199
5200     return -d $testdir;
5201 }
5202
5203 sub pci_dev_group_bind_to_vfio {
5204     my ($pciid) = @_;
5205
5206     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5207
5208     if (!-d $vfio_basedir) {
5209         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5210     }
5211     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5212
5213     # get IOMMU group devices
5214     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5215       my @devs = grep /^0000:/, readdir($D);
5216     closedir($D);
5217
5218     foreach my $pciid (@devs) {
5219         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5220
5221         # pci bridges, switches or root ports are not supported
5222         # they have a pci_bus subdirectory so skip them
5223         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5224
5225         my $info = pci_device_info($1);
5226         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5227     }
5228
5229     return 1;
5230 }
5231
5232 # vzdump restore implementaion
5233
5234 sub tar_archive_read_firstfile {
5235     my $archive = shift;
5236
5237     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5238
5239     # try to detect archive type first
5240     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5241         die "unable to open file '$archive'\n";
5242     my $firstfile = <$fh>;
5243     kill 15, $pid;
5244     close $fh;
5245
5246     die "ERROR: archive contaions no data\n" if !$firstfile;
5247     chomp $firstfile;
5248
5249     return $firstfile;
5250 }
5251
5252 sub tar_restore_cleanup {
5253     my ($storecfg, $statfile) = @_;
5254
5255     print STDERR "starting cleanup\n";
5256
5257     if (my $fd = IO::File->new($statfile, "r")) {
5258         while (defined(my $line = <$fd>)) {
5259             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5260                 my $volid = $2;
5261                 eval {
5262                     if ($volid =~ m|^/|) {
5263                         unlink $volid || die 'unlink failed\n';
5264                     } else {
5265                         PVE::Storage::vdisk_free($storecfg, $volid);
5266                     }
5267                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5268                 };
5269                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5270             } else {
5271                 print STDERR "unable to parse line in statfile - $line";
5272             }
5273         }
5274         $fd->close();
5275     }
5276 }
5277
5278 sub restore_archive {
5279     my ($archive, $vmid, $user, $opts) = @_;
5280
5281     my $format = $opts->{format};
5282     my $comp;
5283
5284     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5285         $format = 'tar' if !$format;
5286         $comp = 'gzip';
5287     } elsif ($archive =~ m/\.tar$/) {
5288         $format = 'tar' if !$format;
5289     } elsif ($archive =~ m/.tar.lzo$/) {
5290         $format = 'tar' if !$format;
5291         $comp = 'lzop';
5292     } elsif ($archive =~ m/\.vma$/) {
5293         $format = 'vma' if !$format;
5294     } elsif ($archive =~ m/\.vma\.gz$/) {
5295         $format = 'vma' if !$format;
5296         $comp = 'gzip';
5297     } elsif ($archive =~ m/\.vma\.lzo$/) {
5298         $format = 'vma' if !$format;
5299         $comp = 'lzop';
5300     } else {
5301         $format = 'vma' if !$format; # default
5302     }
5303
5304     # try to detect archive format
5305     if ($format eq 'tar') {
5306         return restore_tar_archive($archive, $vmid, $user, $opts);
5307     } else {
5308         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5309     }
5310 }
5311
5312 sub restore_update_config_line {
5313     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5314
5315     return if $line =~ m/^\#qmdump\#/;
5316     return if $line =~ m/^\#vzdump\#/;
5317     return if $line =~ m/^lock:/;
5318     return if $line =~ m/^unused\d+:/;
5319     return if $line =~ m/^parent:/;
5320     return if $line =~ m/^template:/; # restored VM is never a template
5321
5322     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5323     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5324         # try to convert old 1.X settings
5325         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5326         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5327             my ($model, $macaddr) = split(/\=/, $devconfig);
5328             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5329             my $net = {
5330                 model => $model,
5331                 bridge => "vmbr$ind",
5332                 macaddr => $macaddr,
5333             };
5334             my $netstr = print_net($net);
5335
5336             print $outfd "net$cookie->{netcount}: $netstr\n";
5337             $cookie->{netcount}++;
5338         }
5339     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5340         my ($id, $netstr) = ($1, $2);
5341         my $net = parse_net($netstr);
5342         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5343         $netstr = print_net($net);
5344         print $outfd "$id: $netstr\n";
5345     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5346         my $virtdev = $1;
5347         my $value = $3;
5348         my $di = parse_drive($virtdev, $value);
5349         if (defined($di->{backup}) && !$di->{backup}) {
5350             print $outfd "#$line";
5351         } elsif ($map->{$virtdev}) {
5352             delete $di->{format}; # format can change on restore
5353             $di->{file} = $map->{$virtdev};
5354             $value = print_drive($vmid, $di);
5355             print $outfd "$virtdev: $value\n";
5356         } else {
5357             print $outfd $line;
5358         }
5359     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5360         my ($uuid, $uuid_str);
5361         UUID::generate($uuid);
5362         UUID::unparse($uuid, $uuid_str);
5363         my $smbios1 = parse_smbios1($2);
5364         $smbios1->{uuid} = $uuid_str;
5365         print $outfd $1.print_smbios1($smbios1)."\n";
5366     } else {
5367         print $outfd $line;
5368     }
5369 }
5370
5371 sub scan_volids {
5372     my ($cfg, $vmid) = @_;
5373
5374     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5375
5376     my $volid_hash = {};
5377     foreach my $storeid (keys %$info) {
5378         foreach my $item (@{$info->{$storeid}}) {
5379             next if !($item->{volid} && $item->{size});
5380             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5381             $volid_hash->{$item->{volid}} = $item;
5382         }
5383     }
5384
5385     return $volid_hash;
5386 }
5387
5388 sub is_volume_in_use {
5389     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5390
5391     my $path = PVE::Storage::path($storecfg, $volid);
5392
5393     my $scan_config = sub {
5394         my ($cref, $snapname) = @_;
5395
5396         foreach my $key (keys %$cref) {
5397             my $value = $cref->{$key};
5398             if (is_valid_drivename($key)) {
5399                 next if $skip_drive && $key eq $skip_drive;
5400                 my $drive = parse_drive($key, $value);
5401                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5402                 return 1 if $volid eq $drive->{file};
5403                 if ($drive->{file} =~ m!^/!) {
5404                     return 1 if $drive->{file} eq $path;
5405                 } else {
5406                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5407                     next if !$storeid;
5408                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5409                     next if !$scfg;
5410                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5411                 }
5412             }
5413         }
5414
5415         return 0;
5416     };
5417
5418     return 1 if &$scan_config($conf);
5419
5420     undef $skip_drive;
5421
5422     foreach my $snapname (keys %{$conf->{snapshots}}) {
5423         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5424     }
5425
5426     return 0;
5427 }
5428
5429 sub update_disksize {
5430     my ($vmid, $conf, $volid_hash) = @_;
5431
5432     my $changes;
5433
5434     # used and unused disks
5435     my $referenced = {};
5436
5437     # Note: it is allowed to define multiple storages with same path (alias), so
5438     # we need to check both 'volid' and real 'path' (two different volid can point
5439     # to the same path).
5440
5441     my $referencedpath = {};
5442
5443     # update size info
5444     foreach my $opt (keys %$conf) {
5445         if (is_valid_drivename($opt)) {
5446             my $drive = parse_drive($opt, $conf->{$opt});
5447             my $volid = $drive->{file};
5448             next if !$volid;
5449
5450             $referenced->{$volid} = 1;
5451             if ($volid_hash->{$volid} &&
5452                 (my $path = $volid_hash->{$volid}->{path})) {
5453                 $referencedpath->{$path} = 1;
5454             }
5455
5456             next if drive_is_cdrom($drive);
5457             next if !$volid_hash->{$volid};
5458
5459             $drive->{size} = $volid_hash->{$volid}->{size};
5460             my $new = print_drive($vmid, $drive);
5461             if ($new ne $conf->{$opt}) {
5462                 $changes = 1;
5463                 $conf->{$opt} = $new;
5464             }
5465         }
5466     }
5467
5468     # remove 'unusedX' entry if volume is used
5469     foreach my $opt (keys %$conf) {
5470         next if $opt !~ m/^unused\d+$/;
5471         my $volid = $conf->{$opt};
5472         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5473         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5474             $changes = 1;
5475             delete $conf->{$opt};
5476         }
5477
5478         $referenced->{$volid} = 1;
5479         $referencedpath->{$path} = 1 if $path;
5480     }
5481
5482     foreach my $volid (sort keys %$volid_hash) {
5483         next if $volid =~ m/vm-$vmid-state-/;
5484         next if $referenced->{$volid};
5485         my $path = $volid_hash->{$volid}->{path};
5486         next if !$path; # just to be sure
5487         next if $referencedpath->{$path};
5488         $changes = 1;
5489         PVE::QemuConfig->add_unused_volume($conf, $volid);
5490         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5491     }
5492
5493     return $changes;
5494 }
5495
5496 sub rescan {
5497     my ($vmid, $nolock) = @_;
5498
5499     my $cfg = PVE::Storage::config();
5500
5501     my $volid_hash = scan_volids($cfg, $vmid);
5502
5503     my $updatefn =  sub {
5504         my ($vmid) = @_;
5505
5506         my $conf = PVE::QemuConfig->load_config($vmid);
5507
5508         PVE::QemuConfig->check_lock($conf);
5509
5510         my $vm_volids = {};
5511         foreach my $volid (keys %$volid_hash) {
5512             my $info = $volid_hash->{$volid};
5513             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5514         }
5515
5516         my $changes = update_disksize($vmid, $conf, $vm_volids);
5517
5518         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5519     };
5520
5521     if (defined($vmid)) {
5522         if ($nolock) {
5523             &$updatefn($vmid);
5524         } else {
5525             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5526         }
5527     } else {
5528         my $vmlist = config_list();
5529         foreach my $vmid (keys %$vmlist) {
5530             if ($nolock) {
5531                 &$updatefn($vmid);
5532             } else {
5533                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5534             }
5535         }
5536     }
5537 }
5538
5539 sub restore_vma_archive {
5540     my ($archive, $vmid, $user, $opts, $comp) = @_;
5541
5542     my $input = $archive eq '-' ? "<&STDIN" : undef;
5543     my $readfrom = $archive;
5544
5545     my $uncomp = '';
5546     if ($comp) {
5547         $readfrom = '-';
5548         my $qarchive = PVE::Tools::shellquote($archive);
5549         if ($comp eq 'gzip') {
5550             $uncomp = "zcat $qarchive|";
5551         } elsif ($comp eq 'lzop') {
5552             $uncomp = "lzop -d -c $qarchive|";
5553         } else {
5554             die "unknown compression method '$comp'\n";
5555         }
5556
5557     }
5558
5559     my $tmpdir = "/var/tmp/vzdumptmp$$";
5560     rmtree $tmpdir;
5561
5562     # disable interrupts (always do cleanups)
5563     local $SIG{INT} =
5564         local $SIG{TERM} =
5565         local $SIG{QUIT} =
5566         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5567
5568     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5569     POSIX::mkfifo($mapfifo, 0600);
5570     my $fifofh;
5571
5572     my $openfifo = sub {
5573         open($fifofh, '>', $mapfifo) || die $!;
5574     };
5575
5576     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5577
5578     my $oldtimeout;
5579     my $timeout = 5;
5580
5581     my $devinfo = {};
5582
5583     my $rpcenv = PVE::RPCEnvironment::get();
5584
5585     my $conffile = PVE::QemuConfig->config_file($vmid);
5586     my $tmpfn = "$conffile.$$.tmp";
5587
5588     # Note: $oldconf is undef if VM does not exists
5589     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5590     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5591
5592     my $print_devmap = sub {
5593         my $virtdev_hash = {};
5594
5595         my $cfgfn = "$tmpdir/qemu-server.conf";
5596
5597         # we can read the config - that is already extracted
5598         my $fh = IO::File->new($cfgfn, "r") ||
5599             "unable to read qemu-server.conf - $!\n";
5600
5601         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5602         if (-f $fwcfgfn) {
5603             my $pve_firewall_dir = '/etc/pve/firewall';
5604             mkdir $pve_firewall_dir; # make sure the dir exists
5605             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5606         }
5607
5608         while (defined(my $line = <$fh>)) {
5609             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5610                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5611                 die "archive does not contain data for drive '$virtdev'\n"
5612                     if !$devinfo->{$devname};
5613                 if (defined($opts->{storage})) {
5614                     $storeid = $opts->{storage} || 'local';
5615                 } elsif (!$storeid) {
5616                     $storeid = 'local';
5617                 }
5618                 $format = 'raw' if !$format;
5619                 $devinfo->{$devname}->{devname} = $devname;
5620                 $devinfo->{$devname}->{virtdev} = $virtdev;
5621                 $devinfo->{$devname}->{format} = $format;
5622                 $devinfo->{$devname}->{storeid} = $storeid;
5623
5624                 # check permission on storage
5625                 my $pool = $opts->{pool}; # todo: do we need that?
5626                 if ($user ne 'root@pam') {
5627                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5628                 }
5629
5630                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5631             }
5632         }
5633
5634         foreach my $devname (keys %$devinfo) {
5635             die "found no device mapping information for device '$devname'\n"
5636                 if !$devinfo->{$devname}->{virtdev};
5637         }
5638
5639         my $cfg = PVE::Storage::config();
5640
5641         # create empty/temp config
5642         if ($oldconf) {
5643             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5644             foreach_drive($oldconf, sub {
5645                 my ($ds, $drive) = @_;
5646
5647                 return if drive_is_cdrom($drive);
5648
5649                 my $volid = $drive->{file};
5650
5651                 return if !$volid || $volid =~ m|^/|;
5652
5653                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5654                 return if !$path || !$owner || ($owner != $vmid);
5655
5656                 # Note: only delete disk we want to restore
5657                 # other volumes will become unused
5658                 if ($virtdev_hash->{$ds}) {
5659                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5660                     if (my $err = $@) {
5661                         warn $err;
5662                     }
5663                 }
5664             });
5665
5666             # delete vmstate files
5667             # since after the restore we have no snapshots anymore
5668             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5669                 my $snap = $oldconf->{snapshots}->{$snapname};
5670                 if ($snap->{vmstate}) {
5671                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5672                     if (my $err = $@) {
5673                         warn $err;
5674                     }
5675                 }
5676             }
5677         }
5678
5679         my $map = {};
5680         foreach my $virtdev (sort keys %$virtdev_hash) {
5681             my $d = $virtdev_hash->{$virtdev};
5682             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5683             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5684
5685             # test if requested format is supported
5686             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5687             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5688             $d->{format} = $defFormat if !$supported;
5689
5690             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5691                                                   $d->{format}, undef, $alloc_size);
5692             print STDERR "new volume ID is '$volid'\n";
5693             $d->{volid} = $volid;
5694             my $path = PVE::Storage::path($cfg, $volid);
5695
5696             PVE::Storage::activate_volumes($cfg,[$volid]);
5697
5698             my $write_zeros = 1;
5699             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5700                 $write_zeros = 0;
5701             }
5702
5703             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5704
5705             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5706             $map->{$virtdev} = $volid;
5707         }
5708
5709         $fh->seek(0, 0) || die "seek failed - $!\n";
5710
5711         my $outfd = new IO::File ($tmpfn, "w") ||
5712             die "unable to write config for VM $vmid\n";
5713
5714         my $cookie = { netcount => 0 };
5715         while (defined(my $line = <$fh>)) {
5716             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5717         }
5718
5719         $fh->close();
5720         $outfd->close();
5721     };
5722
5723     eval {
5724         # enable interrupts
5725         local $SIG{INT} =
5726             local $SIG{TERM} =
5727             local $SIG{QUIT} =
5728             local $SIG{HUP} =
5729             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5730         local $SIG{ALRM} = sub { die "got timeout\n"; };
5731
5732         $oldtimeout = alarm($timeout);
5733
5734         my $parser = sub {
5735             my $line = shift;
5736
5737             print "$line\n";
5738
5739             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5740                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5741                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5742             } elsif ($line =~ m/^CTIME: /) {
5743                 # we correctly received the vma config, so we can disable
5744                 # the timeout now for disk allocation (set to 10 minutes, so
5745                 # that we always timeout if something goes wrong)
5746                 alarm(600);
5747                 &$print_devmap();
5748                 print $fifofh "done\n";
5749                 my $tmp = $oldtimeout || 0;
5750                 $oldtimeout = undef;
5751                 alarm($tmp);
5752                 close($fifofh);
5753             }
5754         };
5755
5756         print "restore vma archive: $cmd\n";
5757         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5758     };
5759     my $err = $@;
5760
5761     alarm($oldtimeout) if $oldtimeout;
5762
5763     my $vollist = [];
5764     foreach my $devname (keys %$devinfo) {
5765         my $volid = $devinfo->{$devname}->{volid};
5766         push @$vollist, $volid if $volid;
5767     }
5768
5769     my $cfg = PVE::Storage::config();
5770     PVE::Storage::deactivate_volumes($cfg, $vollist);
5771
5772     unlink $mapfifo;
5773
5774     if ($err) {
5775         rmtree $tmpdir;
5776         unlink $tmpfn;
5777
5778         foreach my $devname (keys %$devinfo) {
5779             my $volid = $devinfo->{$devname}->{volid};
5780             next if !$volid;
5781             eval {
5782                 if ($volid =~ m|^/|) {
5783                     unlink $volid || die 'unlink failed\n';
5784                 } else {
5785                     PVE::Storage::vdisk_free($cfg, $volid);
5786                 }
5787                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5788             };
5789             print STDERR "unable to cleanup '$volid' - $@" if $@;
5790         }
5791         die $err;
5792     }
5793
5794     rmtree $tmpdir;
5795
5796     rename($tmpfn, $conffile) ||
5797         die "unable to commit configuration file '$conffile'\n";
5798
5799     PVE::Cluster::cfs_update(); # make sure we read new file
5800
5801     eval { rescan($vmid, 1); };
5802     warn $@ if $@;
5803 }
5804
5805 sub restore_tar_archive {
5806     my ($archive, $vmid, $user, $opts) = @_;
5807
5808     if ($archive ne '-') {
5809         my $firstfile = tar_archive_read_firstfile($archive);
5810         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5811             if $firstfile ne 'qemu-server.conf';
5812     }
5813
5814     my $storecfg = PVE::Storage::config();
5815
5816     # destroy existing data - keep empty config
5817     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5818     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5819
5820     my $tocmd = "/usr/lib/qemu-server/qmextract";
5821
5822     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5823     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5824     $tocmd .= ' --prealloc' if $opts->{prealloc};
5825     $tocmd .= ' --info' if $opts->{info};
5826
5827     # tar option "xf" does not autodetect compression when read from STDIN,
5828     # so we pipe to zcat
5829     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5830         PVE::Tools::shellquote("--to-command=$tocmd");
5831
5832     my $tmpdir = "/var/tmp/vzdumptmp$$";
5833     mkpath $tmpdir;
5834
5835     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5836     local $ENV{VZDUMP_VMID} = $vmid;
5837     local $ENV{VZDUMP_USER} = $user;
5838
5839     my $conffile = PVE::QemuConfig->config_file($vmid);
5840     my $tmpfn = "$conffile.$$.tmp";
5841
5842     # disable interrupts (always do cleanups)
5843     local $SIG{INT} =
5844         local $SIG{TERM} =
5845         local $SIG{QUIT} =
5846         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5847
5848     eval {
5849         # enable interrupts
5850         local $SIG{INT} =
5851             local $SIG{TERM} =
5852             local $SIG{QUIT} =
5853             local $SIG{HUP} =
5854             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5855
5856         if ($archive eq '-') {
5857             print "extracting archive from STDIN\n";
5858             run_command($cmd, input => "<&STDIN");
5859         } else {
5860             print "extracting archive '$archive'\n";
5861             run_command($cmd);
5862         }
5863
5864         return if $opts->{info};
5865
5866         # read new mapping
5867         my $map = {};
5868         my $statfile = "$tmpdir/qmrestore.stat";
5869         if (my $fd = IO::File->new($statfile, "r")) {
5870             while (defined (my $line = <$fd>)) {
5871                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5872                     $map->{$1} = $2 if $1;
5873                 } else {
5874                     print STDERR "unable to parse line in statfile - $line\n";
5875                 }
5876             }
5877             $fd->close();
5878         }
5879
5880         my $confsrc = "$tmpdir/qemu-server.conf";
5881
5882         my $srcfd = new IO::File($confsrc, "r") ||
5883             die "unable to open file '$confsrc'\n";
5884
5885         my $outfd = new IO::File ($tmpfn, "w") ||
5886             die "unable to write config for VM $vmid\n";
5887
5888         my $cookie = { netcount => 0 };
5889         while (defined (my $line = <$srcfd>)) {
5890             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5891         }
5892
5893         $srcfd->close();
5894         $outfd->close();
5895     };
5896     my $err = $@;
5897
5898     if ($err) {
5899
5900         unlink $tmpfn;
5901
5902         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5903
5904         die $err;
5905     }
5906
5907     rmtree $tmpdir;
5908
5909     rename $tmpfn, $conffile ||
5910         die "unable to commit configuration file '$conffile'\n";
5911
5912     PVE::Cluster::cfs_update(); # make sure we read new file
5913
5914     eval { rescan($vmid, 1); };
5915     warn $@ if $@;
5916 };
5917
5918 sub foreach_storage_used_by_vm {
5919     my ($conf, $func) = @_;
5920
5921     my $sidhash = {};
5922
5923     foreach_drive($conf, sub {
5924         my ($ds, $drive) = @_;
5925         return if drive_is_cdrom($drive);
5926
5927         my $volid = $drive->{file};
5928
5929         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5930         $sidhash->{$sid} = $sid if $sid;
5931     });
5932
5933     foreach my $sid (sort keys %$sidhash) {
5934         &$func($sid);
5935     }
5936 }
5937
5938 sub do_snapshots_with_qemu {
5939     my ($storecfg, $volid) = @_;
5940
5941     my $storage_name = PVE::Storage::parse_volume_id($volid);
5942
5943     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5944         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5945         return 1;
5946     }
5947
5948     if ($volid =~ m/\.(qcow2|qed)$/){
5949         return 1;
5950     }
5951
5952     return undef;
5953 }
5954
5955 sub qga_check_running {
5956     my ($vmid) = @_;
5957
5958     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5959     if ($@) {
5960         warn "Qemu Guest Agent is not running - $@";
5961         return 0;
5962     }
5963     return 1;
5964 }
5965
5966 sub template_create {
5967     my ($vmid, $conf, $disk) = @_;
5968
5969     my $storecfg = PVE::Storage::config();
5970
5971     foreach_drive($conf, sub {
5972         my ($ds, $drive) = @_;
5973
5974         return if drive_is_cdrom($drive);
5975         return if $disk && $ds ne $disk;
5976
5977         my $volid = $drive->{file};
5978         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5979
5980         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5981         $drive->{file} = $voliddst;
5982         $conf->{$ds} = print_drive($vmid, $drive);
5983         PVE::QemuConfig->write_config($vmid, $conf);
5984     });
5985 }
5986
5987 sub qemu_img_convert {
5988     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5989
5990     my $storecfg = PVE::Storage::config();
5991     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5992     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5993
5994     if ($src_storeid && $dst_storeid) {
5995
5996         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5997
5998         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5999         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6000
6001         my $src_format = qemu_img_format($src_scfg, $src_volname);
6002         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6003
6004         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6005         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6006
6007         my $cmd = [];
6008         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6009         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6010         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6011         if ($is_zero_initialized) {
6012             push @$cmd, "zeroinit:$dst_path";
6013         } else {
6014             push @$cmd, $dst_path;
6015         }
6016
6017         my $parser = sub {
6018             my $line = shift;
6019             if($line =~ m/\((\S+)\/100\%\)/){
6020                 my $percent = $1;
6021                 my $transferred = int($size * $percent / 100);
6022                 my $remaining = $size - $transferred;
6023
6024                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6025             }
6026
6027         };
6028
6029         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6030         my $err = $@;
6031         die "copy failed: $err" if $err;
6032     }
6033 }
6034
6035 sub qemu_img_format {
6036     my ($scfg, $volname) = @_;
6037
6038     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6039         return $1;
6040     } else {
6041         return "raw";
6042     }
6043 }
6044
6045 sub qemu_drive_mirror {
6046     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6047
6048     $jobs = {} if !$jobs;
6049
6050     my $qemu_target;
6051     my $format;
6052     $jobs->{"drive-$drive"} = {};
6053
6054     if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
6055         my $server = $1;
6056         my $port = $2;
6057         my $exportname = $3;
6058
6059         $format = "nbd";
6060         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
6061         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
6062         my $cmd = ['socat', '-T30', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=5"];
6063
6064         my $pid = fork();
6065         if (!defined($pid)) {
6066             die "forking socat tunnel failed\n";
6067         } elsif ($pid == 0) {
6068             exec(@$cmd);
6069             warn "exec failed: $!\n";
6070             POSIX::_exit(-1);
6071         }
6072         $jobs->{"drive-$drive"}->{pid} = $pid;
6073
6074         my $timeout = 0;
6075         while (!-S $unixsocket) {
6076             die "nbd connection helper timed out\n"
6077                 if $timeout++ > 5;
6078             sleep 1;
6079         }
6080     } else {
6081         my $storecfg = PVE::Storage::config();
6082         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6083
6084         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6085
6086         $format = qemu_img_format($dst_scfg, $dst_volname);
6087
6088         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6089
6090         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6091     }
6092
6093     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6094     $opts->{format} = $format if $format;
6095
6096     print "drive mirror is starting for drive-$drive\n";
6097
6098     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6099
6100     if (my $err = $@) {
6101         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6102         die "mirroring error: $err";
6103     }
6104
6105     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6106 }
6107
6108 sub qemu_drive_mirror_monitor {
6109     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6110
6111     eval {
6112         my $err_complete = 0;
6113
6114         while (1) {
6115             die "storage migration timed out\n" if $err_complete > 300;
6116
6117             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6118
6119             my $running_mirror_jobs = {};
6120             foreach my $stat (@$stats) {
6121                 next if $stat->{type} ne 'mirror';
6122                 $running_mirror_jobs->{$stat->{device}} = $stat;
6123             }
6124
6125             my $readycounter = 0;
6126
6127             foreach my $job (keys %$jobs) {
6128
6129                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6130                     print "$job : finished\n";
6131                     delete $jobs->{$job};
6132                     next;
6133                 }
6134
6135                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6136
6137                 my $busy = $running_mirror_jobs->{$job}->{busy};
6138                 my $ready = $running_mirror_jobs->{$job}->{ready};
6139                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6140                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6141                     my $remaining = $total - $transferred;
6142                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6143
6144                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6145                 }
6146
6147                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6148             }
6149
6150             last if scalar(keys %$jobs) == 0;
6151
6152             if ($readycounter == scalar(keys %$jobs)) {
6153                 print "all mirroring jobs are ready \n";
6154                 last if $skipcomplete; #do the complete later
6155
6156                 if ($vmiddst && $vmiddst != $vmid) {
6157                     my $agent_running = $qga && qga_check_running($vmid);
6158                     if ($agent_running) {
6159                         print "freeze filesystem\n";
6160                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6161                     } else {
6162                         print "suspend vm\n";
6163                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6164                     }
6165
6166                     # if we clone a disk for a new target vm, we don't switch the disk
6167                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6168
6169                     if ($agent_running) {
6170                         print "unfreeze filesystem\n";
6171                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6172                     } else {
6173                         print "resume vm\n";
6174                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6175                     }
6176
6177                     last;
6178                 } else {
6179
6180                     foreach my $job (keys %$jobs) {
6181                         # try to switch the disk if source and destination are on the same guest
6182                         print "$job: Completing block job...\n";
6183
6184                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6185                         if ($@ =~ m/cannot be completed/) {
6186                             print "$job: Block job cannot be completed, try again.\n";
6187                             $err_complete++;
6188                         }else {
6189                             print "$job: Completed successfully.\n";
6190                             $jobs->{$job}->{complete} = 1;
6191                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6192                         }
6193                     }
6194                 }
6195             }
6196             sleep 1;
6197         }
6198     };
6199     my $err = $@;
6200
6201     if ($err) {
6202         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6203         die "mirroring error: $err";
6204     }
6205
6206 }
6207
6208 sub qemu_blockjobs_cancel {
6209     my ($vmid, $jobs) = @_;
6210
6211     foreach my $job (keys %$jobs) {
6212         print "$job: Cancelling block job\n";
6213         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6214         $jobs->{$job}->{cancel} = 1;
6215     }
6216
6217     while (1) {
6218         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6219
6220         my $running_jobs = {};
6221         foreach my $stat (@$stats) {
6222             $running_jobs->{$stat->{device}} = $stat;
6223         }
6224
6225         foreach my $job (keys %$jobs) {
6226
6227             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6228                 print "$job: Done.\n";
6229                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6230                 delete $jobs->{$job};
6231             }
6232         }
6233
6234         last if scalar(keys %$jobs) == 0;
6235
6236         sleep 1;
6237     }
6238 }
6239
6240 sub qemu_blockjobs_finish_tunnel {
6241    my ($vmid, $job, $cpid) = @_;
6242
6243    return if !$cpid;
6244
6245    for (my $i = 1; $i < 20; $i++) {
6246         my $waitpid = waitpid($cpid, WNOHANG);
6247         last if (defined($waitpid) && ($waitpid == $cpid));
6248
6249         if ($i == 10) {
6250             kill(15, $cpid);
6251         } elsif ($i >= 15) {
6252             kill(9, $cpid);
6253         }
6254         sleep (1);
6255     }
6256     unlink "/run/qemu-server/$vmid.mirror-$job";
6257 }
6258
6259 sub clone_disk {
6260     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6261         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6262
6263     my $newvolid;
6264
6265     if (!$full) {
6266         print "create linked clone of drive $drivename ($drive->{file})\n";
6267         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6268         push @$newvollist, $newvolid;
6269     } else {
6270
6271         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6272         $storeid = $storage if $storage;
6273
6274         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6275         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6276
6277         print "create full clone of drive $drivename ($drive->{file})\n";
6278         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, undef, ($size/1024));
6279         push @$newvollist, $newvolid;
6280
6281         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6282
6283         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6284         if (!$running || $snapname) {
6285             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6286         } else {
6287
6288             my $kvmver = get_running_qemu_version ($vmid);
6289             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6290                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6291                     if $drive->{iothread};
6292             }
6293
6294             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6295         }
6296     }
6297
6298     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6299
6300     my $disk = $drive;
6301     $disk->{format} = undef;
6302     $disk->{file} = $newvolid;
6303     $disk->{size} = $size;
6304
6305     return $disk;
6306 }
6307
6308 # this only works if VM is running
6309 sub get_current_qemu_machine {
6310     my ($vmid) = @_;
6311
6312     my $cmd = { execute => 'query-machines', arguments => {} };
6313     my $res = vm_qmp_command($vmid, $cmd);
6314
6315     my ($current, $default);
6316     foreach my $e (@$res) {
6317         $default = $e->{name} if $e->{'is-default'};
6318         $current = $e->{name} if $e->{'is-current'};
6319     }
6320
6321     # fallback to the default machine if current is not supported by qemu
6322     return $current || $default || 'pc';
6323 }
6324
6325 sub get_running_qemu_version {
6326     my ($vmid) = @_;
6327     my $cmd = { execute => 'query-version', arguments => {} };
6328     my $res = vm_qmp_command($vmid, $cmd);
6329     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6330 }
6331
6332 sub qemu_machine_feature_enabled {
6333     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6334
6335     my $current_major;
6336     my $current_minor;
6337
6338     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6339
6340         $current_major = $3;
6341         $current_minor = $4;
6342
6343     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6344
6345         $current_major = $1;
6346         $current_minor = $2;
6347     }
6348
6349     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6350
6351
6352 }
6353
6354 sub qemu_machine_pxe {
6355     my ($vmid, $conf, $machine) = @_;
6356
6357     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6358
6359     foreach my $opt (keys %$conf) {
6360         next if $opt !~ m/^net(\d+)$/;
6361         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6362         next if !$net;
6363         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6364         return $machine.".pxe" if $romfile =~ m/pxe/;
6365         last;
6366     }
6367
6368     return $machine;
6369 }
6370
6371 sub qemu_use_old_bios_files {
6372     my ($machine_type) = @_;
6373
6374     return if !$machine_type;
6375
6376     my $use_old_bios_files = undef;
6377
6378     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6379         $machine_type = $1;
6380         $use_old_bios_files = 1;
6381     } else {
6382         my $kvmver = kvm_user_version();
6383         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6384         # load new efi bios files on migration. So this hack is required to allow
6385         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6386         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6387         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6388     }
6389
6390     return ($use_old_bios_files, $machine_type);
6391 }
6392
6393 sub create_efidisk {
6394     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6395
6396     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6397
6398     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6399     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6400     PVE::Storage::activate_volumes($storecfg, [$volid]);
6401
6402     my $path = PVE::Storage::path($storecfg, $volid);
6403     eval {
6404         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6405     };
6406     die "Copying EFI vars image failed: $@" if $@;
6407
6408     return ($volid, $vars_size);
6409 }
6410
6411 sub lspci {
6412
6413     my $devices = {};
6414
6415     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6416             my (undef, $id, $function) = @_;
6417             my $res = { id => $id, function => $function};
6418             push @{$devices->{$id}}, $res;
6419     });
6420
6421     # Entries should be sorted by functions.
6422     foreach my $id (keys %$devices) {
6423         my $dev = $devices->{$id};
6424         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6425     }
6426
6427     return $devices;
6428 }
6429
6430 sub vm_iothreads_list {
6431     my ($vmid) = @_;
6432
6433     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6434
6435     my $iothreads = {};
6436     foreach my $iothread (@$res) {
6437         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6438     }
6439
6440     return $iothreads;
6441 }
6442
6443 sub scsihw_infos {
6444     my ($conf, $drive) = @_;
6445
6446     my $maxdev = 0;
6447
6448     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6449         $maxdev = 7;
6450     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6451         $maxdev = 1;
6452     } else {
6453         $maxdev = 256;
6454     }
6455
6456     my $controller = int($drive->{index} / $maxdev);
6457     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6458
6459     return ($maxdev, $controller, $controller_prefix);
6460 }
6461
6462 sub add_hyperv_enlightenments {
6463     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6464
6465     return if $winversion < 6;
6466     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6467
6468     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6469
6470     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6471         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6472         push @$cpuFlags , 'hv_vapic';
6473         push @$cpuFlags , 'hv_time';
6474     } else {
6475         push @$cpuFlags , 'hv_spinlocks=0xffff';
6476     }
6477
6478     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6479         push @$cpuFlags , 'hv_reset';
6480         push @$cpuFlags , 'hv_vpindex';
6481         push @$cpuFlags , 'hv_runtime';
6482     }
6483
6484     if ($winversion >= 7) {
6485         push @$cpuFlags , 'hv_relaxed';
6486     }
6487 }
6488
6489 sub windows_version {
6490     my ($ostype) = @_;
6491
6492     return 0 if !$ostype;
6493
6494     my $winversion = 0;
6495
6496     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6497         $winversion = 5;
6498     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6499         $winversion = 6;
6500     } elsif ($ostype =~ m/^win(\d+)$/) {
6501         $winversion = $1;
6502     }
6503
6504     return $winversion;
6505 }
6506
6507 sub resolve_dst_disk_format {
6508         my ($storecfg, $storeid, $src_volname, $format) = @_;
6509         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6510
6511         if (!$format) {
6512             # if no target format is specified, use the source disk format as hint
6513             if ($src_volname) {
6514                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6515                 $format = qemu_img_format($scfg, $src_volname);
6516             } else {
6517                 return $defFormat;
6518             }
6519         }
6520
6521         # test if requested format is supported - else use default
6522         my $supported = grep { $_ eq $format } @$validFormats;
6523         $format = $defFormat if !$supported;
6524         return $format;
6525 }
6526
6527 sub resolve_first_disk {
6528     my $conf = shift;
6529     my @disks = PVE::QemuServer::valid_drive_names();
6530     my $firstdisk;
6531     foreach my $ds (reverse @disks) {
6532         next if !$conf->{$ds};
6533         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6534         next if PVE::QemuServer::drive_is_cdrom($disk);
6535         $firstdisk = $ds;
6536     }
6537     return $firstdisk;
6538 }
6539
6540 sub generate_smbios1_uuid {
6541     my ($uuid, $uuid_str);
6542     UUID::generate($uuid);
6543     UUID::unparse($uuid, $uuid_str);
6544     return "uuid=$uuid_str";
6545 }
6546
6547 # bash completion helper
6548
6549 sub complete_backup_archives {
6550     my ($cmdname, $pname, $cvalue) = @_;
6551
6552     my $cfg = PVE::Storage::config();
6553
6554     my $storeid;
6555
6556     if ($cvalue =~ m/^([^:]+):/) {
6557         $storeid = $1;
6558     }
6559
6560     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6561
6562     my $res = [];
6563     foreach my $id (keys %$data) {
6564         foreach my $item (@{$data->{$id}}) {
6565             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6566             push @$res, $item->{volid} if defined($item->{volid});
6567         }
6568     }
6569
6570     return $res;
6571 }
6572
6573 my $complete_vmid_full = sub {
6574     my ($running) = @_;
6575
6576     my $idlist = vmstatus();
6577
6578     my $res = [];
6579
6580     foreach my $id (keys %$idlist) {
6581         my $d = $idlist->{$id};
6582         if (defined($running)) {
6583             next if $d->{template};
6584             next if $running && $d->{status} ne 'running';
6585             next if !$running && $d->{status} eq 'running';
6586         }
6587         push @$res, $id;
6588
6589     }
6590     return $res;
6591 };
6592
6593 sub complete_vmid {
6594     return &$complete_vmid_full();
6595 }
6596
6597 sub complete_vmid_stopped {
6598     return &$complete_vmid_full(0);
6599 }
6600
6601 sub complete_vmid_running {
6602     return &$complete_vmid_full(1);
6603 }
6604
6605 sub complete_storage {
6606
6607     my $cfg = PVE::Storage::config();
6608     my $ids = $cfg->{ids};
6609
6610     my $res = [];
6611     foreach my $sid (keys %$ids) {
6612         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6613         next if !$ids->{$sid}->{content}->{images};
6614         push @$res, $sid;
6615     }
6616
6617     return $res;
6618 }
6619
6620 sub nbd_stop {
6621     my ($vmid) = @_;
6622
6623     vm_mon_cmd($vmid, 'nbd-server-stop');
6624 }
6625
6626 1;