PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42
  43 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  44
  45 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  46
  47 # Note about locking: we use flock on the config file protect
  48 # against concurent actions.
  49 # Aditionaly, we have a 'lock' setting in the config file. This
  50 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  51 # allowed when such lock is set. But you can ignore this kind of
  52 # lock with the --skiplock flag.
  53
  54 cfs_register_file('/qemu-server/',
  55                   \&parse_vm_config,
  56                   \&write_vm_config);
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  72     type => 'string',
  73     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  74     description => "The drive's backing file's data format.",
  75     optional => 1,
  76 });
  77
  78 #no warnings 'redefine';
  79
  80 sub cgroups_write {
  81    my ($controller, $vmid, $option, $value) = @_;
  82
  83    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  84    PVE::ProcFSTools::write_proc_entry($path, $value);
  85
  86 }
  87
  88 my $nodename = PVE::INotify::nodename();
  89
  90 mkdir "/etc/pve/nodes/$nodename";
  91 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  92 mkdir $confdir;
  93
  94 my $var_run_tmpdir = "/var/run/qemu-server";
  95 mkdir $var_run_tmpdir;
  96
  97 my $lock_dir = "/var/lock/qemu-server";
  98 mkdir $lock_dir;
  99
 100 my $pcisysfs = "/sys/bus/pci";
 101
 102 my $cpu_vendor_list = {
 103     # Intel CPUs
 104     486 => 'GenuineIntel',
 105     pentium => 'GenuineIntel',
 106     pentium2  => 'GenuineIntel',
 107     pentium3  => 'GenuineIntel',
 108     coreduo => 'GenuineIntel',
 109     core2duo => 'GenuineIntel',
 110     Conroe  => 'GenuineIntel',
 111     Penryn  => 'GenuineIntel',
 112     Nehalem  => 'GenuineIntel',
 113     'Nehalem-IBRS'  => 'GenuineIntel',
 114     Westmere => 'GenuineIntel',
 115     'Westmere-IBRS' => 'GenuineIntel',
 116     SandyBridge => 'GenuineIntel',
 117     'SandyBridge-IBRS' => 'GenuineIntel',
 118     IvyBridge => 'GenuineIntel',
 119     'IvyBridge-IBRS' => 'GenuineIntel',
 120     Haswell => 'GenuineIntel',
 121     'Haswell-IBRS' => 'GenuineIntel',
 122     'Haswell-noTSX' => 'GenuineIntel',
 123     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 124     Broadwell => 'GenuineIntel',
 125     'Broadwell-IBRS' => 'GenuineIntel',
 126     'Broadwell-noTSX' => 'GenuineIntel',
 127     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 128     'Skylake-Client' => 'GenuineIntel',
 129     'Skylake-Client-IBRS' => 'GenuineIntel',
 130
 131     # AMD CPUs
 132     athlon => 'AuthenticAMD',
 133     phenom  => 'AuthenticAMD',
 134     Opteron_G1  => 'AuthenticAMD',
 135     Opteron_G2  => 'AuthenticAMD',
 136     Opteron_G3  => 'AuthenticAMD',
 137     Opteron_G4  => 'AuthenticAMD',
 138     Opteron_G5  => 'AuthenticAMD',
 139
 140     # generic types, use vendor from host node
 141     host => 'default',
 142     kvm32 => 'default',
 143     kvm64 => 'default',
 144     qemu32 => 'default',
 145     qemu64 => 'default',
 146 };
 147
 148 my $cpu_fmt = {
 149     cputype => {
 150         description => "Emulated CPU type.",
 151         type => 'string',
 152         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 153         default => 'kvm64',
 154         default_key => 1,
 155     },
 156     hidden => {
 157         description => "Do not identify as a KVM virtual machine.",
 158         type => 'boolean',
 159         optional => 1,
 160         default => 0
 161     },
 162     flags => {
 163         description => "Override CPU flags. Currently only the 'pcid' flag is supported."
 164                      . " Use '+pcid' or '-pcid' to enable or disable."
 165                      . " This takes precedence over flags coming from the cpu type or changed implicitly via the OS type.",
 166         format_description => 'flaglist',
 167         type => 'string',
 168         pattern => '[+-]pcid',
 169         optional => 1,
 170         default => '',
 171     },
 172 };
 173
 174 my $watchdog_fmt = {
 175     model => {
 176         default_key => 1,
 177         type => 'string',
 178         enum => [qw(i6300esb ib700)],
 179         description => "Watchdog type to emulate.",
 180         default => 'i6300esb',
 181         optional => 1,
 182     },
 183     action => {
 184         type => 'string',
 185         enum => [qw(reset shutdown poweroff pause debug none)],
 186         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 187         optional => 1,
 188     },
 189 };
 190 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 191
 192 my $confdesc = {
 193     onboot => {
 194         optional => 1,
 195         type => 'boolean',
 196         description => "Specifies whether a VM will be started during system bootup.",
 197         default => 0,
 198     },
 199     autostart => {
 200         optional => 1,
 201         type => 'boolean',
 202         description => "Automatic restart after crash (currently ignored).",
 203         default => 0,
 204     },
 205     hotplug => {
 206         optional => 1,
 207         type => 'string', format => 'pve-hotplug-features',
 208         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 209         default => 'network,disk,usb',
 210     },
 211     reboot => {
 212         optional => 1,
 213         type => 'boolean',
 214         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 215         default => 1,
 216     },
 217     lock => {
 218         optional => 1,
 219         type => 'string',
 220         description => "Lock/unlock the VM.",
 221         enum => [qw(migrate backup snapshot rollback)],
 222     },
 223     cpulimit => {
 224         optional => 1,
 225         type => 'number',
 226         description => "Limit of CPU usage.",
 227         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 228         minimum => 0,
 229         maximum => 128,
 230         default => 0,
 231     },
 232     cpuunits => {
 233         optional => 1,
 234         type => 'integer',
 235         description => "CPU weight for a VM.",
 236         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 237         minimum => 2,
 238         maximum => 262144,
 239         default => 1024,
 240     },
 241     memory => {
 242         optional => 1,
 243         type => 'integer',
 244         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 245         minimum => 16,
 246         default => 512,
 247     },
 248     balloon => {
 249         optional => 1,
 250         type => 'integer',
 251         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 252         minimum => 0,
 253     },
 254     shares => {
 255         optional => 1,
 256         type => 'integer',
 257         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 258         minimum => 0,
 259         maximum => 50000,
 260         default => 1000,
 261     },
 262     keyboard => {
 263         optional => 1,
 264         type => 'string',
 265         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.".
 266                        "It should not be necessary to set it.",
 267         enum => PVE::Tools::kvmkeymaplist(),
 268         default => undef,
 269     },
 270     name => {
 271         optional => 1,
 272         type => 'string', format => 'dns-name',
 273         description => "Set a name for the VM. Only used on the configuration web interface.",
 274     },
 275     scsihw => {
 276         optional => 1,
 277         type => 'string',
 278         description => "SCSI controller model",
 279         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 280         default => 'lsi',
 281     },
 282     description => {
 283         optional => 1,
 284         type => 'string',
 285         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 286     },
 287     ostype => {
 288         optional => 1,
 289         type => 'string',
 290         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 291         description => "Specify guest operating system.",
 292         verbose_description => <<EODESC,
 293 Specify guest operating system. This is used to enable special
 294 optimization/features for specific operating systems:
 295
 296 [horizontal]
 297 other;; unspecified OS
 298 wxp;; Microsoft Windows XP
 299 w2k;; Microsoft Windows 2000
 300 w2k3;; Microsoft Windows 2003
 301 w2k8;; Microsoft Windows 2008
 302 wvista;; Microsoft Windows Vista
 303 win7;; Microsoft Windows 7
 304 win8;; Microsoft Windows 8/2012/2012r2
 305 win10;; Microsoft Windows 10/2016
 306 l24;; Linux 2.4 Kernel
 307 l26;; Linux 2.6/3.X Kernel
 308 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 309 EODESC
 310     },
 311     boot => {
 312         optional => 1,
 313         type => 'string',
 314         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 315         pattern => '[acdn]{1,4}',
 316         default => 'cdn',
 317     },
 318     bootdisk => {
 319         optional => 1,
 320         type => 'string', format => 'pve-qm-bootdisk',
 321         description => "Enable booting from specified disk.",
 322         pattern => '(ide|sata|scsi|virtio)\d+',
 323     },
 324     smp => {
 325         optional => 1,
 326         type => 'integer',
 327         description => "The number of CPUs. Please use option -sockets instead.",
 328         minimum => 1,
 329         default => 1,
 330     },
 331     sockets => {
 332         optional => 1,
 333         type => 'integer',
 334         description => "The number of CPU sockets.",
 335         minimum => 1,
 336         default => 1,
 337     },
 338     cores => {
 339         optional => 1,
 340         type => 'integer',
 341         description => "The number of cores per socket.",
 342         minimum => 1,
 343         default => 1,
 344     },
 345     numa => {
 346         optional => 1,
 347         type => 'boolean',
 348         description => "Enable/disable NUMA.",
 349         default => 0,
 350     },
 351     hugepages => {
 352         optional => 1,
 353         type => 'string',
 354         description => "Enable/disable hugepages memory.",
 355         enum => [qw(any 2 1024)],
 356     },
 357     vcpus => {
 358         optional => 1,
 359         type => 'integer',
 360         description => "Number of hotplugged vcpus.",
 361         minimum => 1,
 362         default => 0,
 363     },
 364     acpi => {
 365         optional => 1,
 366         type => 'boolean',
 367         description => "Enable/disable ACPI.",
 368         default => 1,
 369     },
 370     agent => {
 371         optional => 1,
 372         type => 'boolean',
 373         description => "Enable/disable Qemu GuestAgent.",
 374         default => 0,
 375     },
 376     kvm => {
 377         optional => 1,
 378         type => 'boolean',
 379         description => "Enable/disable KVM hardware virtualization.",
 380         default => 1,
 381     },
 382     tdf => {
 383         optional => 1,
 384         type => 'boolean',
 385         description => "Enable/disable time drift fix.",
 386         default => 0,
 387     },
 388     localtime => {
 389         optional => 1,
 390         type => 'boolean',
 391         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 392     },
 393     freeze => {
 394         optional => 1,
 395         type => 'boolean',
 396         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 397     },
 398     vga => {
 399         optional => 1,
 400         type => 'string',
 401         description => "Select the VGA type.",
 402         verbose_description => "Select the VGA type. If you want to use high resolution" .
 403             " modes (>= 1280x1024x16) then you should use the options " .
 404             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 405             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 406             "display sever. For win* OS you can select how many independent " .
 407             "displays you want, Linux guests can add displays them self. " .
 408             "You can also run without any graphic card, using a serial device" .
 409             " as terminal.",
 410         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 411     },
 412     watchdog => {
 413         optional => 1,
 414         type => 'string', format => 'pve-qm-watchdog',
 415         description => "Create a virtual hardware watchdog device.",
 416         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 417             " (by a guest action), the watchdog must be periodically polled " .
 418             "by an agent inside the guest or else the watchdog will reset " .
 419             "the guest (or execute the respective action specified)",
 420     },
 421     startdate => {
 422         optional => 1,
 423         type => 'string',
 424         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 425         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 426         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 427         default => 'now',
 428     },
 429     startup =>  get_standard_option('pve-startup-order'),
 430     template => {
 431         optional => 1,
 432         type => 'boolean',
 433         description => "Enable/disable Template.",
 434         default => 0,
 435     },
 436     args => {
 437         optional => 1,
 438         type => 'string',
 439         description => "Arbitrary arguments passed to kvm.",
 440         verbose_description => <<EODESCR,
 441 Arbitrary arguments passed to kvm, for example:
 442
 443 args: -no-reboot -no-hpet
 444
 445 NOTE: this option is for experts only.
 446 EODESCR
 447     },
 448     tablet => {
 449         optional => 1,
 450         type => 'boolean',
 451         default => 1,
 452         description => "Enable/disable the USB tablet device.",
 453         verbose_description => "Enable/disable the USB tablet device. This device is " .
 454             "usually needed to allow absolute mouse positioning with VNC. " .
 455             "Else the mouse runs out of sync with normal VNC clients. " .
 456             "If you're running lots of console-only guests on one host, " .
 457             "you may consider disabling this to save some context switches. " .
 458             "This is turned off by default if you use spice (-vga=qxl).",
 459     },
 460     migrate_speed => {
 461         optional => 1,
 462         type => 'integer',
 463         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 464         minimum => 0,
 465         default => 0,
 466     },
 467     migrate_downtime => {
 468         optional => 1,
 469         type => 'number',
 470         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 471         minimum => 0,
 472         default => 0.1,
 473     },
 474     cdrom => {
 475         optional => 1,
 476         type => 'string', format => 'pve-qm-ide',
 477         typetext => '<volume>',
 478         description => "This is an alias for option -ide2",
 479     },
 480     cpu => {
 481         optional => 1,
 482         description => "Emulated CPU type.",
 483         type => 'string',
 484         format => $cpu_fmt,
 485     },
 486     parent => get_standard_option('pve-snapshot-name', {
 487         optional => 1,
 488         description => "Parent snapshot name. This is used internally, and should not be modified.",
 489     }),
 490     snaptime => {
 491         optional => 1,
 492         description => "Timestamp for snapshots.",
 493         type => 'integer',
 494         minimum => 0,
 495     },
 496     vmstate => {
 497         optional => 1,
 498         type => 'string', format => 'pve-volume-id',
 499         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 500     },
 501     vmstatestorage => get_standard_option('pve-storage-id', {
 502         description => "Default storage for VM state volumes/files.",
 503         optional => 1,
 504     }),
 505     machine => {
 506         description => "Specific the Qemu machine type.",
 507         type => 'string',
 508         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 509         maxLength => 40,
 510         optional => 1,
 511     },
 512     smbios1 => {
 513         description => "Specify SMBIOS type 1 fields.",
 514         type => 'string', format => 'pve-qm-smbios1',
 515         maxLength => 256,
 516         optional => 1,
 517     },
 518     protection => {
 519         optional => 1,
 520         type => 'boolean',
 521         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 522         default => 0,
 523     },
 524     bios => {
 525         optional => 1,
 526         type => 'string',
 527         enum => [ qw(seabios ovmf) ],
 528         description => "Select BIOS implementation.",
 529         default => 'seabios',
 530     },
 531 };
 532
 533 # what about other qemu settings ?
 534 #cpu => 'string',
 535 #machine => 'string',
 536 #fda => 'file',
 537 #fdb => 'file',
 538 #mtdblock => 'file',
 539 #sd => 'file',
 540 #pflash => 'file',
 541 #snapshot => 'bool',
 542 #bootp => 'file',
 543 ##tftp => 'dir',
 544 ##smb => 'dir',
 545 #kernel => 'file',
 546 #append => 'string',
 547 #initrd => 'file',
 548 ##soundhw => 'string',
 549
 550 while (my ($k, $v) = each %$confdesc) {
 551     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 552 }
 553
 554 my $MAX_IDE_DISKS = 4;
 555 my $MAX_SCSI_DISKS = 14;
 556 my $MAX_VIRTIO_DISKS = 16;
 557 my $MAX_SATA_DISKS = 6;
 558 my $MAX_USB_DEVICES = 5;
 559 my $MAX_NETS = 32;
 560 my $MAX_UNUSED_DISKS = 8;
 561 my $MAX_HOSTPCI_DEVICES = 4;
 562 my $MAX_SERIAL_PORTS = 4;
 563 my $MAX_PARALLEL_PORTS = 3;
 564 my $MAX_NUMA = 8;
 565
 566 my $numa_fmt = {
 567     cpus => {
 568         type => "string",
 569         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 570         description => "CPUs accessing this NUMA node.",
 571         format_description => "id[-id];...",
 572     },
 573     memory => {
 574         type => "number",
 575         description => "Amount of memory this NUMA node provides.",
 576         optional => 1,
 577     },
 578     hostnodes => {
 579         type => "string",
 580         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 581         description => "Host NUMA nodes to use.",
 582         format_description => "id[-id];...",
 583         optional => 1,
 584     },
 585     policy => {
 586         type => 'string',
 587         enum => [qw(preferred bind interleave)],
 588         description => "NUMA allocation policy.",
 589         optional => 1,
 590     },
 591 };
 592 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 593 my $numadesc = {
 594     optional => 1,
 595     type => 'string', format => $numa_fmt,
 596     description => "NUMA topology.",
 597 };
 598 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 599
 600 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 601     $confdesc->{"numa$i"} = $numadesc;
 602 }
 603
 604 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 605                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 606                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 607 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 608
 609 my $net_fmt_bridge_descr = <<__EOD__;
 610 Bridge to attach the network device to. The Proxmox VE standard bridge
 611 is called 'vmbr0'.
 612
 613 If you do not specify a bridge, we create a kvm user (NATed) network
 614 device, which provides DHCP and DNS services. The following addresses
 615 are used:
 616
 617  10.0.2.2   Gateway
 618  10.0.2.3   DNS Server
 619  10.0.2.4   SMB Server
 620
 621 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 622 __EOD__
 623
 624 my $net_fmt = {
 625     macaddr => {
 626         type => 'string',
 627         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 628         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 629         format_description => "XX:XX:XX:XX:XX:XX",
 630         optional => 1,
 631     },
 632     model => {
 633         type => 'string',
 634         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 635         enum => $nic_model_list,
 636         default_key => 1,
 637     },
 638     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 639     bridge => {
 640         type => 'string',
 641         description => $net_fmt_bridge_descr,
 642         format_description => 'bridge',
 643         optional => 1,
 644     },
 645     queues => {
 646         type => 'integer',
 647         minimum => 0, maximum => 16,
 648         description => 'Number of packet queues to be used on the device.',
 649         optional => 1,
 650     },
 651     rate => {
 652         type => 'number',
 653         minimum => 0,
 654         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 655         optional => 1,
 656     },
 657     tag => {
 658         type => 'integer',
 659         minimum => 1, maximum => 4094,
 660         description => 'VLAN tag to apply to packets on this interface.',
 661         optional => 1,
 662     },
 663     trunks => {
 664         type => 'string',
 665         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 666         description => 'VLAN trunks to pass through this interface.',
 667         format_description => 'vlanid[;vlanid...]',
 668         optional => 1,
 669     },
 670     firewall => {
 671         type => 'boolean',
 672         description => 'Whether this interface should be protected by the firewall.',
 673         optional => 1,
 674     },
 675     link_down => {
 676         type => 'boolean',
 677         description => 'Whether this interface should be disconnected (like pulling the plug).',
 678         optional => 1,
 679     },
 680 };
 681
 682 my $netdesc = {
 683     optional => 1,
 684     type => 'string', format => $net_fmt,
 685     description => "Specify network devices.",
 686 };
 687
 688 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 689
 690 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 691     $confdesc->{"net$i"} = $netdesc;
 692 }
 693
 694 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 695 sub verify_volume_id_or_qm_path {
 696     my ($volid, $noerr) = @_;
 697
 698     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 699         return $volid;
 700     }
 701
 702     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 703     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 704     if ($@) {
 705         return undef if $noerr;
 706         die $@;
 707     }
 708     return $volid;
 709 }
 710
 711 my $drivename_hash;
 712
 713 my %drivedesc_base = (
 714     volume => { alias => 'file' },
 715     file => {
 716         type => 'string',
 717         format => 'pve-volume-id-or-qm-path',
 718         default_key => 1,
 719         format_description => 'volume',
 720         description => "The drive's backing volume.",
 721     },
 722     media => {
 723         type => 'string',
 724         enum => [qw(cdrom disk)],
 725         description => "The drive's media type.",
 726         default => 'disk',
 727         optional => 1
 728     },
 729     cyls => {
 730         type => 'integer',
 731         description => "Force the drive's physical geometry to have a specific cylinder count.",
 732         optional => 1
 733     },
 734     heads => {
 735         type => 'integer',
 736         description => "Force the drive's physical geometry to have a specific head count.",
 737         optional => 1
 738     },
 739     secs => {
 740         type => 'integer',
 741         description => "Force the drive's physical geometry to have a specific sector count.",
 742         optional => 1
 743     },
 744     trans => {
 745         type => 'string',
 746         enum => [qw(none lba auto)],
 747         description => "Force disk geometry bios translation mode.",
 748         optional => 1,
 749     },
 750     snapshot => {
 751         type => 'boolean',
 752         description => "Whether the drive should be included when making snapshots.",
 753         optional => 1,
 754     },
 755     cache => {
 756         type => 'string',
 757         enum => [qw(none writethrough writeback unsafe directsync)],
 758         description => "The drive's cache mode",
 759         optional => 1,
 760     },
 761     format => get_standard_option('pve-qm-image-format'),
 762     size => {
 763         type => 'string',
 764         format => 'disk-size',
 765         format_description => 'DiskSize',
 766         description => "Disk size. This is purely informational and has no effect.",
 767         optional => 1,
 768     },
 769     backup => {
 770         type => 'boolean',
 771         description => "Whether the drive should be included when making backups.",
 772         optional => 1,
 773     },
 774     replicate => {
 775         type => 'boolean',
 776         description => 'Whether the drive should considered for replication jobs.',
 777         optional => 1,
 778         default => 1,
 779     },
 780     rerror => {
 781         type => 'string',
 782         enum => [qw(ignore report stop)],
 783         description => 'Read error action.',
 784         optional => 1,
 785     },
 786     werror => {
 787         type => 'string',
 788         enum => [qw(enospc ignore report stop)],
 789         description => 'Write error action.',
 790         optional => 1,
 791     },
 792     aio => {
 793         type => 'string',
 794         enum => [qw(native threads)],
 795         description => 'AIO type to use.',
 796         optional => 1,
 797     },
 798     discard => {
 799         type => 'string',
 800         enum => [qw(ignore on)],
 801         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 802         optional => 1,
 803     },
 804     detect_zeroes => {
 805         type => 'boolean',
 806         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 807         optional => 1,
 808     },
 809     serial => {
 810         type => 'string',
 811         format => 'urlencoded',
 812         format_description => 'serial',
 813         maxLength => 20*3, # *3 since it's %xx url enoded
 814         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 815         optional => 1,
 816     }
 817 );
 818
 819 my %iothread_fmt = ( iothread => {
 820         type => 'boolean',
 821         description => "Whether to use iothreads for this drive",
 822         optional => 1,
 823 });
 824
 825 my %model_fmt = (
 826     model => {
 827         type => 'string',
 828         format => 'urlencoded',
 829         format_description => 'model',
 830         maxLength => 40*3, # *3 since it's %xx url enoded
 831         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 832         optional => 1,
 833     },
 834 );
 835
 836 my %queues_fmt = (
 837     queues => {
 838         type => 'integer',
 839         description => "Number of queues.",
 840         minimum => 2,
 841         optional => 1
 842     }
 843 );
 844
 845 my %scsiblock_fmt = (
 846     scsiblock => {
 847         type => 'boolean',
 848         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 849         optional => 1,
 850         default => 0,
 851     },
 852 );
 853
 854 my $add_throttle_desc = sub {
 855     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 856     my $d = {
 857         type => $type,
 858         format_description => $unit,
 859         description => "Maximum $what in $longunit.",
 860         optional => 1,
 861     };
 862     $d->{minimum} = $minimum if defined($minimum);
 863     $drivedesc_base{$key} = $d;
 864 };
 865 # throughput: (leaky bucket)
 866 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 867 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 868 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 869 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 870 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 871 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 872 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 873 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 874 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 875
 876 # pools: (pool of IO before throttling starts taking effect)
 877 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 878 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 879 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 880 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 881 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 882 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 883
 884 # burst lengths
 885 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 886 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 887 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 888 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 889 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 890 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 891
 892 # legacy support
 893 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
 894 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
 895 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
 896 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
 897
 898 my $ide_fmt = {
 899     %drivedesc_base,
 900     %model_fmt,
 901 };
 902 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 903
 904 my $idedesc = {
 905     optional => 1,
 906     type => 'string', format => $ide_fmt,
 907     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 908 };
 909 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 910
 911 my $scsi_fmt = {
 912     %drivedesc_base,
 913     %iothread_fmt,
 914     %queues_fmt,
 915     %scsiblock_fmt,
 916 };
 917 my $scsidesc = {
 918     optional => 1,
 919     type => 'string', format => $scsi_fmt,
 920     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 921 };
 922 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 923
 924 my $sata_fmt = {
 925     %drivedesc_base,
 926 };
 927 my $satadesc = {
 928     optional => 1,
 929     type => 'string', format => $sata_fmt,
 930     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 931 };
 932 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 933
 934 my $virtio_fmt = {
 935     %drivedesc_base,
 936     %iothread_fmt,
 937 };
 938 my $virtiodesc = {
 939     optional => 1,
 940     type => 'string', format => $virtio_fmt,
 941     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 942 };
 943 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 944
 945 my $alldrive_fmt = {
 946     %drivedesc_base,
 947     %iothread_fmt,
 948     %model_fmt,
 949     %queues_fmt,
 950     %scsiblock_fmt,
 951 };
 952
 953 my $efidisk_fmt = {
 954     volume => { alias => 'file' },
 955     file => {
 956         type => 'string',
 957         format => 'pve-volume-id-or-qm-path',
 958         default_key => 1,
 959         format_description => 'volume',
 960         description => "The drive's backing volume.",
 961     },
 962     format => get_standard_option('pve-qm-image-format'),
 963     size => {
 964         type => 'string',
 965         format => 'disk-size',
 966         format_description => 'DiskSize',
 967         description => "Disk size. This is purely informational and has no effect.",
 968         optional => 1,
 969     },
 970 };
 971
 972 my $efidisk_desc = {
 973     optional => 1,
 974     type => 'string', format => $efidisk_fmt,
 975     description => "Configure a Disk for storing EFI vars",
 976 };
 977
 978 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 979
 980 my $usb_fmt = {
 981     host => {
 982         default_key => 1,
 983         type => 'string', format => 'pve-qm-usb-device',
 984         format_description => 'HOSTUSBDEVICE|spice',
 985         description => <<EODESCR,
 986 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 987
 988  'bus-port(.port)*' (decimal numbers) or
 989  'vendor_id:product_id' (hexadeciaml numbers) or
 990  'spice'
 991
 992 You can use the 'lsusb -t' command to list existing usb devices.
 993
 994 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 995
 996 The value 'spice' can be used to add a usb redirection devices for spice.
 997 EODESCR
 998     },
 999     usb3 => {
1000         optional => 1,
1001         type => 'boolean',
1002         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1003         default => 0,
1004     },
1005 };
1006
1007 my $usbdesc = {
1008     optional => 1,
1009     type => 'string', format => $usb_fmt,
1010     description => "Configure an USB device (n is 0 to 4).",
1011 };
1012 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1013
1014 # NOTE: the match-groups of this regex are used in parse_hostpci
1015 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1016 my $hostpci_fmt = {
1017     host => {
1018         default_key => 1,
1019         type => 'string',
1020         pattern => qr/$PCIRE(;$PCIRE)*/,
1021         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1022         description => <<EODESCR,
1023 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1024 of PCI virtual functions of the host. HOSTPCIID syntax is:
1025
1026 'bus:dev.func' (hexadecimal numbers)
1027
1028 You can us the 'lspci' command to list existing PCI devices.
1029 EODESCR
1030     },
1031     rombar => {
1032         type => 'boolean',
1033         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1034         optional => 1,
1035         default => 1,
1036     },
1037     romfile => {
1038         type => 'string',
1039         pattern => '[^,;]+',
1040         format_description => 'string',
1041         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1042         optional => 1,
1043     },
1044     pcie => {
1045         type => 'boolean',
1046         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1047         optional => 1,
1048         default => 0,
1049     },
1050     'x-vga' => {
1051         type => 'boolean',
1052         description =>  "Enable vfio-vga device support.",
1053         optional => 1,
1054         default => 0,
1055     },
1056 };
1057 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1058
1059 my $hostpcidesc = {
1060         optional => 1,
1061         type => 'string', format => 'pve-qm-hostpci',
1062         description => "Map host PCI devices into guest.",
1063         verbose_description =>  <<EODESCR,
1064 Map host PCI devices into guest.
1065
1066 NOTE: This option allows direct access to host hardware. So it is no longer
1067 possible to migrate such machines - use with special care.
1068
1069 CAUTION: Experimental! User reported problems with this option.
1070 EODESCR
1071 };
1072 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1073
1074 my $serialdesc = {
1075         optional => 1,
1076         type => 'string',
1077         pattern => '(/dev/.+|socket)',
1078         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1079         verbose_description =>  <<EODESCR,
1080 Create a serial device inside the VM (n is 0 to 3), and pass through a
1081 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1082 host side (use 'qm terminal' to open a terminal connection).
1083
1084 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1085
1086 CAUTION: Experimental! User reported problems with this option.
1087 EODESCR
1088 };
1089
1090 my $paralleldesc= {
1091         optional => 1,
1092         type => 'string',
1093         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1094         description =>  "Map host parallel devices (n is 0 to 2).",
1095         verbose_description =>  <<EODESCR,
1096 Map host parallel devices (n is 0 to 2).
1097
1098 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1099
1100 CAUTION: Experimental! User reported problems with this option.
1101 EODESCR
1102 };
1103
1104 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1105     $confdesc->{"parallel$i"} = $paralleldesc;
1106 }
1107
1108 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1109     $confdesc->{"serial$i"} = $serialdesc;
1110 }
1111
1112 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1113     $confdesc->{"hostpci$i"} = $hostpcidesc;
1114 }
1115
1116 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1117     $drivename_hash->{"ide$i"} = 1;
1118     $confdesc->{"ide$i"} = $idedesc;
1119 }
1120
1121 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1122     $drivename_hash->{"sata$i"} = 1;
1123     $confdesc->{"sata$i"} = $satadesc;
1124 }
1125
1126 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1127     $drivename_hash->{"scsi$i"} = 1;
1128     $confdesc->{"scsi$i"} = $scsidesc ;
1129 }
1130
1131 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1132     $drivename_hash->{"virtio$i"} = 1;
1133     $confdesc->{"virtio$i"} = $virtiodesc;
1134 }
1135
1136 $drivename_hash->{efidisk0} = 1;
1137 $confdesc->{efidisk0} = $efidisk_desc;
1138
1139 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1140     $confdesc->{"usb$i"} = $usbdesc;
1141 }
1142
1143 my $unuseddesc = {
1144     optional => 1,
1145     type => 'string', format => 'pve-volume-id',
1146     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1147 };
1148
1149 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1150     $confdesc->{"unused$i"} = $unuseddesc;
1151 }
1152
1153 my $kvm_api_version = 0;
1154
1155 sub kvm_version {
1156
1157     return $kvm_api_version if $kvm_api_version;
1158
1159     my $fh = IO::File->new("</dev/kvm") ||
1160         return 0;
1161
1162     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1163         $kvm_api_version = $v;
1164     }
1165
1166     $fh->close();
1167
1168     return  $kvm_api_version;
1169 }
1170
1171 my $kvm_user_version;
1172
1173 sub kvm_user_version {
1174
1175     return $kvm_user_version if $kvm_user_version;
1176
1177     $kvm_user_version = 'unknown';
1178
1179     my $code = sub {
1180         my $line = shift;
1181         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1182             $kvm_user_version = $2;
1183         }
1184     };
1185
1186     eval { run_command("kvm -version", outfunc => $code); };
1187     warn $@ if $@;
1188
1189     return $kvm_user_version;
1190
1191 }
1192
1193 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1194
1195 sub valid_drive_names {
1196     # order is important - used to autoselect boot disk
1197     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1198             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1199             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1200             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1201             'efidisk0');
1202 }
1203
1204 sub is_valid_drivename {
1205     my $dev = shift;
1206
1207     return defined($drivename_hash->{$dev});
1208 }
1209
1210 sub option_exists {
1211     my $key = shift;
1212     return defined($confdesc->{$key});
1213 }
1214
1215 sub nic_models {
1216     return $nic_model_list;
1217 }
1218
1219 sub os_list_description {
1220
1221     return {
1222         other => 'Other',
1223         wxp => 'Windows XP',
1224         w2k => 'Windows 2000',
1225         w2k3 =>, 'Windows 2003',
1226         w2k8 => 'Windows 2008',
1227         wvista => 'Windows Vista',
1228         win7 => 'Windows 7',
1229         win8 => 'Windows 8/2012',
1230         win10 => 'Windows 10/2016',
1231         l24 => 'Linux 2.4',
1232         l26 => 'Linux 2.6',
1233     };
1234 }
1235
1236 my $cdrom_path;
1237
1238 sub get_cdrom_path {
1239
1240     return  $cdrom_path if $cdrom_path;
1241
1242     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1243     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1244     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1245 }
1246
1247 sub get_iso_path {
1248     my ($storecfg, $vmid, $cdrom) = @_;
1249
1250     if ($cdrom eq 'cdrom') {
1251         return get_cdrom_path();
1252     } elsif ($cdrom eq 'none') {
1253         return '';
1254     } elsif ($cdrom =~ m|^/|) {
1255         return $cdrom;
1256     } else {
1257         return PVE::Storage::path($storecfg, $cdrom);
1258     }
1259 }
1260
1261 # try to convert old style file names to volume IDs
1262 sub filename_to_volume_id {
1263     my ($vmid, $file, $media) = @_;
1264
1265     if (!($file eq 'none' || $file eq 'cdrom' ||
1266           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1267
1268         return undef if $file =~ m|/|;
1269
1270         if ($media && $media eq 'cdrom') {
1271             $file = "local:iso/$file";
1272         } else {
1273             $file = "local:$vmid/$file";
1274         }
1275     }
1276
1277     return $file;
1278 }
1279
1280 sub verify_media_type {
1281     my ($opt, $vtype, $media) = @_;
1282
1283     return if !$media;
1284
1285     my $etype;
1286     if ($media eq 'disk') {
1287         $etype = 'images';
1288     } elsif ($media eq 'cdrom') {
1289         $etype = 'iso';
1290     } else {
1291         die "internal error";
1292     }
1293
1294     return if ($vtype eq $etype);
1295
1296     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1297 }
1298
1299 sub cleanup_drive_path {
1300     my ($opt, $storecfg, $drive) = @_;
1301
1302     # try to convert filesystem paths to volume IDs
1303
1304     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1305         ($drive->{file} !~ m|^/dev/.+|) &&
1306         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1307         ($drive->{file} !~ m/^\d+$/)) {
1308         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1309         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1310         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1311         verify_media_type($opt, $vtype, $drive->{media});
1312         $drive->{file} = $volid;
1313     }
1314
1315     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1316 }
1317
1318 sub parse_hotplug_features {
1319     my ($data) = @_;
1320
1321     my $res = {};
1322
1323     return $res if $data eq '0';
1324
1325     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1326
1327     foreach my $feature (PVE::Tools::split_list($data)) {
1328         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1329             $res->{$1} = 1;
1330         } else {
1331             die "invalid hotplug feature '$feature'\n";
1332         }
1333     }
1334     return $res;
1335 }
1336
1337 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1338 sub pve_verify_hotplug_features {
1339     my ($value, $noerr) = @_;
1340
1341     return $value if parse_hotplug_features($value);
1342
1343     return undef if $noerr;
1344
1345     die "unable to parse hotplug option\n";
1346 }
1347
1348 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1349 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1350 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1351 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1352 #        [,iothread=on][,serial=serial][,model=model]
1353
1354 sub parse_drive {
1355     my ($key, $data) = @_;
1356
1357     my ($interface, $index);
1358
1359     if ($key =~ m/^([^\d]+)(\d+)$/) {
1360         $interface = $1;
1361         $index = $2;
1362     } else {
1363         return undef;
1364     }
1365
1366     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1367                                      : $confdesc->{$key}->{format};
1368     if (!$desc) {
1369         warn "invalid drive key: $key\n";
1370         return undef;
1371     }
1372     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1373     return undef if !$res;
1374     $res->{interface} = $interface;
1375     $res->{index} = $index;
1376
1377     my $error = 0;
1378     foreach my $opt (qw(bps bps_rd bps_wr)) {
1379         if (my $bps = defined(delete $res->{$opt})) {
1380             if (defined($res->{"m$opt"})) {
1381                 warn "both $opt and m$opt specified\n";
1382                 ++$error;
1383                 next;
1384             }
1385             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1386         }
1387     }
1388
1389     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1390     for my $requirement (
1391         [mbps_max => 'mbps'],
1392         [mbps_rd_max => 'mbps_rd'],
1393         [mbps_wr_max => 'mbps_wr'],
1394         [miops_max => 'miops'],
1395         [miops_rd_max => 'miops_rd'],
1396         [miops_wr_max => 'miops_wr'],
1397         [bps_max_length => 'mbps_max'],
1398         [bps_rd_max_length => 'mbps_rd_max'],
1399         [bps_wr_max_length => 'mbps_wr_max'],
1400         [iops_max_length => 'iops_max'],
1401         [iops_rd_max_length => 'iops_rd_max'],
1402         [iops_wr_max_length => 'iops_wr_max']) {
1403         my ($option, $requires) = @$requirement;
1404         if ($res->{$option} && !$res->{$requires}) {
1405             warn "$option requires $requires\n";
1406             ++$error;
1407         }
1408     }
1409
1410     return undef if $error;
1411
1412     return undef if $res->{mbps_rd} && $res->{mbps};
1413     return undef if $res->{mbps_wr} && $res->{mbps};
1414     return undef if $res->{iops_rd} && $res->{iops};
1415     return undef if $res->{iops_wr} && $res->{iops};
1416
1417     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1418         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1419         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1420         return undef if $res->{interface} eq 'virtio';
1421     }
1422
1423     if (my $size = $res->{size}) {
1424         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1425     }
1426
1427     return $res;
1428 }
1429
1430 sub print_drive {
1431     my ($vmid, $drive) = @_;
1432     my $data = { %$drive };
1433     delete $data->{$_} for qw(index interface);
1434     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1435 }
1436
1437 sub scsi_inquiry {
1438     my($fh, $noerr) = @_;
1439
1440     my $SG_IO = 0x2285;
1441     my $SG_GET_VERSION_NUM = 0x2282;
1442
1443     my $versionbuf = "\x00" x 8;
1444     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1445     if (!$ret) {
1446         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1447         return undef;
1448     }
1449     my $version = unpack("I", $versionbuf);
1450     if ($version < 30000) {
1451         die "scsi generic interface too old\n"  if !$noerr;
1452         return undef;
1453     }
1454
1455     my $buf = "\x00" x 36;
1456     my $sensebuf = "\x00" x 8;
1457     my $cmd = pack("C x3 C x1", 0x12, 36);
1458
1459     # see /usr/include/scsi/sg.h
1460     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1461
1462     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1463                       length($sensebuf), 0, length($buf), $buf,
1464                       $cmd, $sensebuf, 6000);
1465
1466     $ret = ioctl($fh, $SG_IO, $packet);
1467     if (!$ret) {
1468         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1469         return undef;
1470     }
1471
1472     my @res = unpack($sg_io_hdr_t, $packet);
1473     if ($res[17] || $res[18]) {
1474         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1475         return undef;
1476     }
1477
1478     my $res = {};
1479     (my $byte0, my $byte1, $res->{vendor},
1480      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1481
1482     $res->{removable} = $byte1 & 128 ? 1 : 0;
1483     $res->{type} = $byte0 & 31;
1484
1485     return $res;
1486 }
1487
1488 sub path_is_scsi {
1489     my ($path) = @_;
1490
1491     my $fh = IO::File->new("+<$path") || return undef;
1492     my $res = scsi_inquiry($fh, 1);
1493     close($fh);
1494
1495     return $res;
1496 }
1497
1498 sub machine_type_is_q35 {
1499     my ($conf) = @_;
1500
1501     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1502 }
1503
1504 sub print_tabletdevice_full {
1505     my ($conf) = @_;
1506
1507     my $q35 = machine_type_is_q35($conf);
1508
1509     # we use uhci for old VMs because tablet driver was buggy in older qemu
1510     my $usbbus = $q35 ? "ehci" : "uhci";
1511
1512     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1513 }
1514
1515 sub print_drivedevice_full {
1516     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1517
1518     my $device = '';
1519     my $maxdev = 0;
1520
1521     if ($drive->{interface} eq 'virtio') {
1522         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1523         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1524         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1525     } elsif ($drive->{interface} eq 'scsi') {
1526
1527         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1528         my $unit = $drive->{index} % $maxdev;
1529         my $devicetype = 'hd';
1530         my $path = '';
1531         if (drive_is_cdrom($drive)) {
1532             $devicetype = 'cd';
1533         } else {
1534             if ($drive->{file} =~ m|^/|) {
1535                 $path = $drive->{file};
1536                 if (my $info = path_is_scsi($path)) {
1537                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1538                         $devicetype = 'block';
1539                     } elsif ($info->{type} == 1) { # tape
1540                         $devicetype = 'generic';
1541                     }
1542                 }
1543             } else {
1544                  $path = PVE::Storage::path($storecfg, $drive->{file});
1545             }
1546
1547             if($path =~ m/^iscsi\:\/\//){
1548                 $devicetype = 'generic';
1549             }
1550         }
1551
1552         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1553             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1554         } else {
1555             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1556         }
1557
1558     } elsif ($drive->{interface} eq 'ide'){
1559         $maxdev = 2;
1560         my $controller = int($drive->{index} / $maxdev);
1561         my $unit = $drive->{index} % $maxdev;
1562         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1563
1564         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1565         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1566             $model = URI::Escape::uri_unescape($model);
1567             $device .= ",model=$model";
1568         }
1569     } elsif ($drive->{interface} eq 'sata'){
1570         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1571         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1572         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1573     } elsif ($drive->{interface} eq 'usb') {
1574         die "implement me";
1575         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1576     } else {
1577         die "unsupported interface type";
1578     }
1579
1580     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1581
1582     return $device;
1583 }
1584
1585 sub get_initiator_name {
1586     my $initiator;
1587
1588     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1589     while (defined(my $line = <$fh>)) {
1590         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1591         $initiator = $1;
1592         last;
1593     }
1594     $fh->close();
1595
1596     return $initiator;
1597 }
1598
1599 sub print_drive_full {
1600     my ($storecfg, $vmid, $drive) = @_;
1601
1602     my $path;
1603     my $volid = $drive->{file};
1604     my $format;
1605
1606     if (drive_is_cdrom($drive)) {
1607         $path = get_iso_path($storecfg, $vmid, $volid);
1608     } else {
1609         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1610         if ($storeid) {
1611             $path = PVE::Storage::path($storecfg, $volid);
1612             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1613             $format = qemu_img_format($scfg, $volname);
1614         } else {
1615             $path = $volid;
1616             $format = "raw";
1617         }
1618    }
1619
1620     my $opts = '';
1621     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard);
1622     foreach my $o (@qemu_drive_options) {
1623         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1624     }
1625     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1626         my ($dir, $qmpname) = @$type;
1627         if (my $v = $drive->{"mbps$dir"}) {
1628             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1629         }
1630         if (my $v = $drive->{"mbps${dir}_max"}) {
1631             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1632         }
1633         if (my $v = $drive->{"bps${dir}_max_length"}) {
1634             $opts .= ",throttling.bps$qmpname-max-length=$v";
1635         }
1636         if (my $v = $drive->{"iops${dir}"}) {
1637             $opts .= ",throttling.iops$qmpname=$v";
1638         }
1639         if (my $v = $drive->{"iops${dir}_max"}) {
1640             $opts .= ",throttling.iops$qmpname-max=$v";
1641         }
1642         if (my $v = $drive->{"iops${dir}_max_length"}) {
1643             $opts .= ",throttling.iops$qmpname-max-length=$v";
1644         }
1645     }
1646
1647     if (my $serial = $drive->{serial}) {
1648         $serial = URI::Escape::uri_unescape($serial);
1649         $opts .= ",serial=$serial";
1650     }
1651
1652     $opts .= ",format=$format" if $format && !$drive->{format};
1653
1654     my $cache_direct = 0;
1655
1656     if (my $cache = $drive->{cache}) {
1657         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1658     } elsif (!drive_is_cdrom($drive)) {
1659         $opts .= ",cache=none";
1660         $cache_direct = 1;
1661     }
1662
1663     # aio native works only with O_DIRECT
1664     if (!$drive->{aio}) {
1665         if($cache_direct) {
1666             $opts .= ",aio=native";
1667         } else {
1668             $opts .= ",aio=threads";
1669         }
1670     }
1671
1672     if (!drive_is_cdrom($drive)) {
1673         my $detectzeroes;
1674         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1675             $detectzeroes = 'off';
1676         } elsif ($drive->{discard}) {
1677             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1678         } else {
1679             # This used to be our default with discard not being specified:
1680             $detectzeroes = 'on';
1681         }
1682         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1683     }
1684
1685     my $pathinfo = $path ? "file=$path," : '';
1686
1687     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1688 }
1689
1690 sub print_netdevice_full {
1691     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1692
1693     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1694
1695     my $device = $net->{model};
1696     if ($net->{model} eq 'virtio') {
1697          $device = 'virtio-net-pci';
1698      };
1699
1700     my $pciaddr = print_pci_addr("$netid", $bridges);
1701     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1702     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1703         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1704         my $vectors = $net->{queues} * 2 + 2;
1705         $tmpstr .= ",vectors=$vectors,mq=on";
1706     }
1707     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1708
1709     if ($use_old_bios_files) {
1710         my $romfile;
1711         if ($device eq 'virtio-net-pci') {
1712             $romfile = 'pxe-virtio.rom';
1713         } elsif ($device eq 'e1000') {
1714             $romfile = 'pxe-e1000.rom';
1715         } elsif ($device eq 'ne2k') {
1716             $romfile = 'pxe-ne2k_pci.rom';
1717         } elsif ($device eq 'pcnet') {
1718             $romfile = 'pxe-pcnet.rom';
1719         } elsif ($device eq 'rtl8139') {
1720             $romfile = 'pxe-rtl8139.rom';
1721         }
1722         $tmpstr .= ",romfile=$romfile" if $romfile;
1723     }
1724
1725     return $tmpstr;
1726 }
1727
1728 sub print_netdev_full {
1729     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1730
1731     my $i = '';
1732     if ($netid =~ m/^net(\d+)$/) {
1733         $i = int($1);
1734     }
1735
1736     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1737
1738     my $ifname = "tap${vmid}i$i";
1739
1740     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1741     die "interface name '$ifname' is too long (max 15 character)\n"
1742         if length($ifname) >= 16;
1743
1744     my $vhostparam = '';
1745     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1746
1747     my $vmname = $conf->{name} || "vm$vmid";
1748
1749     my $netdev = "";
1750     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1751
1752     if ($net->{bridge}) {
1753         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1754     } else {
1755         $netdev = "type=user,id=$netid,hostname=$vmname";
1756     }
1757
1758     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1759
1760     return $netdev;
1761 }
1762
1763
1764 sub print_cpu_device {
1765     my ($conf, $id) = @_;
1766
1767     my $kvm = $conf->{kvm} // 1;
1768     my $cpu = $kvm ? "kvm64" : "qemu64";
1769     if (my $cputype = $conf->{cpu}) {
1770         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1771             or die "Cannot parse cpu description: $cputype\n";
1772         $cpu = $cpuconf->{cputype};
1773     }
1774
1775     my $cores = $conf->{cores} || 1;
1776
1777     my $current_core = ($id - 1) % $cores;
1778     my $current_socket = int(($id - 1 - $current_core)/$cores);
1779
1780     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1781 }
1782
1783 sub drive_is_cdrom {
1784     my ($drive) = @_;
1785
1786     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1787
1788 }
1789
1790 sub parse_number_sets {
1791     my ($set) = @_;
1792     my $res = [];
1793     foreach my $part (split(/;/, $set)) {
1794         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1795             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1796             push @$res, [ $1, $2 ];
1797         } else {
1798             die "invalid range: $part\n";
1799         }
1800     }
1801     return $res;
1802 }
1803
1804 sub parse_numa {
1805     my ($data) = @_;
1806
1807     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1808     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1809     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1810     return $res;
1811 }
1812
1813 sub parse_hostpci {
1814     my ($value) = @_;
1815
1816     return undef if !$value;
1817
1818     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1819
1820     my @idlist = split(/;/, $res->{host});
1821     delete $res->{host};
1822     foreach my $id (@idlist) {
1823         if ($id =~ /^$PCIRE$/) {
1824             if (defined($2)) {
1825                 push @{$res->{pciid}}, { id => $1, function => $2 };
1826             } else {
1827                 my $pcidevices = lspci($1);
1828                 $res->{pciid} = $pcidevices->{$1};
1829             }
1830         } else {
1831             # should have been caught by parse_property_string already
1832             die "failed to parse PCI id: $id\n";
1833         }
1834     }
1835     return $res;
1836 }
1837
1838 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1839 sub parse_net {
1840     my ($data) = @_;
1841
1842     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1843     if ($@) {
1844         warn $@;
1845         return undef;
1846     }
1847     if (!defined($res->{macaddr})) {
1848         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1849         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1850     }
1851     return $res;
1852 }
1853
1854 sub print_net {
1855     my $net = shift;
1856
1857     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1858 }
1859
1860 sub add_random_macs {
1861     my ($settings) = @_;
1862
1863     foreach my $opt (keys %$settings) {
1864         next if $opt !~ m/^net(\d+)$/;
1865         my $net = parse_net($settings->{$opt});
1866         next if !$net;
1867         $settings->{$opt} = print_net($net);
1868     }
1869 }
1870
1871 sub vm_is_volid_owner {
1872     my ($storecfg, $vmid, $volid) = @_;
1873
1874     if ($volid !~  m|^/|) {
1875         my ($path, $owner);
1876         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1877         if ($owner && ($owner == $vmid)) {
1878             return 1;
1879         }
1880     }
1881
1882     return undef;
1883 }
1884
1885 sub split_flagged_list {
1886     my $text = shift || '';
1887     $text =~ s/[,;]/ /g;
1888     $text =~ s/^\s+//;
1889     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1890 }
1891
1892 sub join_flagged_list {
1893     my ($how, $lst) = @_;
1894     join $how, map { $lst->{$_} . $_ } keys %$lst;
1895 }
1896
1897 sub vmconfig_delete_pending_option {
1898     my ($conf, $key, $force) = @_;
1899
1900     delete $conf->{pending}->{$key};
1901     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1902     $pending_delete_hash->{$key} = $force ? '!' : '';
1903     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1904 }
1905
1906 sub vmconfig_undelete_pending_option {
1907     my ($conf, $key) = @_;
1908
1909     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1910     delete $pending_delete_hash->{$key};
1911
1912     if (%$pending_delete_hash) {
1913         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1914     } else {
1915         delete $conf->{pending}->{delete};
1916     }
1917 }
1918
1919 sub vmconfig_register_unused_drive {
1920     my ($storecfg, $vmid, $conf, $drive) = @_;
1921
1922     if (!drive_is_cdrom($drive)) {
1923         my $volid = $drive->{file};
1924         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1925             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1926         }
1927     }
1928 }
1929
1930 sub vmconfig_cleanup_pending {
1931     my ($conf) = @_;
1932
1933     # remove pending changes when nothing changed
1934     my $changes;
1935     foreach my $opt (keys %{$conf->{pending}}) {
1936         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1937             $changes = 1;
1938             delete $conf->{pending}->{$opt};
1939         }
1940     }
1941
1942     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1943     my $pending_delete_hash = {};
1944     while (my ($opt, $force) = each %$current_delete_hash) {
1945         if (defined($conf->{$opt})) {
1946             $pending_delete_hash->{$opt} = $force;
1947         } else {
1948             $changes = 1;
1949         }
1950     }
1951
1952     if (%$pending_delete_hash) {
1953         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1954     } else {
1955         delete $conf->{pending}->{delete};
1956     }
1957
1958     return $changes;
1959 }
1960
1961 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1962 my $smbios1_fmt = {
1963     uuid => {
1964         type => 'string',
1965         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1966         format_description => 'UUID',
1967         description => "Set SMBIOS1 UUID.",
1968         optional => 1,
1969     },
1970     version => {
1971         type => 'string',
1972         pattern => '\S+',
1973         format_description => 'string',
1974         description => "Set SMBIOS1 version.",
1975         optional => 1,
1976     },
1977     serial => {
1978         type => 'string',
1979         pattern => '\S+',
1980         format_description => 'string',
1981         description => "Set SMBIOS1 serial number.",
1982         optional => 1,
1983     },
1984     manufacturer => {
1985         type => 'string',
1986         pattern => '\S+',
1987         format_description => 'string',
1988         description => "Set SMBIOS1 manufacturer.",
1989         optional => 1,
1990     },
1991     product => {
1992         type => 'string',
1993         pattern => '\S+',
1994         format_description => 'string',
1995         description => "Set SMBIOS1 product ID.",
1996         optional => 1,
1997     },
1998     sku => {
1999         type => 'string',
2000         pattern => '\S+',
2001         format_description => 'string',
2002         description => "Set SMBIOS1 SKU string.",
2003         optional => 1,
2004     },
2005     family => {
2006         type => 'string',
2007         pattern => '\S+',
2008         format_description => 'string',
2009         description => "Set SMBIOS1 family string.",
2010         optional => 1,
2011     },
2012 };
2013
2014 sub parse_smbios1 {
2015     my ($data) = @_;
2016
2017     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2018     warn $@ if $@;
2019     return $res;
2020 }
2021
2022 sub print_smbios1 {
2023     my ($smbios1) = @_;
2024     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2025 }
2026
2027 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2028
2029 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2030 sub verify_bootdisk {
2031     my ($value, $noerr) = @_;
2032
2033     return $value if is_valid_drivename($value);
2034
2035     return undef if $noerr;
2036
2037     die "invalid boot disk '$value'\n";
2038 }
2039
2040 sub parse_watchdog {
2041     my ($value) = @_;
2042
2043     return undef if !$value;
2044
2045     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2046     warn $@ if $@;
2047     return $res;
2048 }
2049
2050 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2051 sub verify_usb_device {
2052     my ($value, $noerr) = @_;
2053
2054     return $value if parse_usb_device($value);
2055
2056     return undef if $noerr;
2057
2058     die "unable to parse usb device\n";
2059 }
2060
2061 # add JSON properties for create and set function
2062 sub json_config_properties {
2063     my $prop = shift;
2064
2065     foreach my $opt (keys %$confdesc) {
2066         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2067         $prop->{$opt} = $confdesc->{$opt};
2068     }
2069
2070     return $prop;
2071 }
2072
2073 sub check_type {
2074     my ($key, $value) = @_;
2075
2076     die "unknown setting '$key'\n" if !$confdesc->{$key};
2077
2078     my $type = $confdesc->{$key}->{type};
2079
2080     if (!defined($value)) {
2081         die "got undefined value\n";
2082     }
2083
2084     if ($value =~ m/[\n\r]/) {
2085         die "property contains a line feed\n";
2086     }
2087
2088     if ($type eq 'boolean') {
2089         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2090         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2091         die "type check ('boolean') failed - got '$value'\n";
2092     } elsif ($type eq 'integer') {
2093         return int($1) if $value =~ m/^(\d+)$/;
2094         die "type check ('integer') failed - got '$value'\n";
2095     } elsif ($type eq 'number') {
2096         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2097         die "type check ('number') failed - got '$value'\n";
2098     } elsif ($type eq 'string') {
2099         if (my $fmt = $confdesc->{$key}->{format}) {
2100             PVE::JSONSchema::check_format($fmt, $value);
2101             return $value;
2102         }
2103         $value =~ s/^\"(.*)\"$/$1/;
2104         return $value;
2105     } else {
2106         die "internal error"
2107     }
2108 }
2109
2110 sub check_iommu_support{
2111     #fixme : need to check IOMMU support
2112     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2113
2114     my $iommu=1;
2115     return $iommu;
2116
2117 }
2118
2119 sub touch_config {
2120     my ($vmid) = @_;
2121
2122     my $conf = PVE::QemuConfig->config_file($vmid);
2123     utime undef, undef, $conf;
2124 }
2125
2126 sub destroy_vm {
2127     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2128
2129     my $conffile = PVE::QemuConfig->config_file($vmid);
2130
2131     my $conf = PVE::QemuConfig->load_config($vmid);
2132
2133     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2134
2135     if ($conf->{template}) {
2136         # check if any base image is still used by a linked clone
2137         foreach_drive($conf, sub {
2138                 my ($ds, $drive) = @_;
2139
2140                 return if drive_is_cdrom($drive);
2141
2142                 my $volid = $drive->{file};
2143
2144                 return if !$volid || $volid =~ m|^/|;
2145
2146                 die "base volume '$volid' is still in use by linked cloned\n"
2147                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2148
2149         });
2150     }
2151
2152     # only remove disks owned by this VM
2153     foreach_drive($conf, sub {
2154         my ($ds, $drive) = @_;
2155
2156         return if drive_is_cdrom($drive);
2157
2158         my $volid = $drive->{file};
2159
2160         return if !$volid || $volid =~ m|^/|;
2161
2162         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2163         return if !$path || !$owner || ($owner != $vmid);
2164
2165         eval {
2166             PVE::Storage::vdisk_free($storecfg, $volid);
2167         };
2168         warn "Could not remove disk '$volid', check manually: $@" if $@;
2169
2170     });
2171
2172     if ($keep_empty_config) {
2173         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2174     } else {
2175         unlink $conffile;
2176     }
2177
2178     # also remove unused disk
2179     eval {
2180         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2181
2182         eval {
2183             PVE::Storage::foreach_volid($dl, sub {
2184                 my ($volid, $sid, $volname, $d) = @_;
2185                 PVE::Storage::vdisk_free($storecfg, $volid);
2186             });
2187         };
2188         warn $@ if $@;
2189
2190     };
2191     warn $@ if $@;
2192 }
2193
2194 sub parse_vm_config {
2195     my ($filename, $raw) = @_;
2196
2197     return undef if !defined($raw);
2198
2199     my $res = {
2200         digest => Digest::SHA::sha1_hex($raw),
2201         snapshots => {},
2202         pending => {},
2203     };
2204
2205     $filename =~ m|/qemu-server/(\d+)\.conf$|
2206         || die "got strange filename '$filename'";
2207
2208     my $vmid = $1;
2209
2210     my $conf = $res;
2211     my $descr;
2212     my $section = '';
2213
2214     my @lines = split(/\n/, $raw);
2215     foreach my $line (@lines) {
2216         next if $line =~ m/^\s*$/;
2217
2218         if ($line =~ m/^\[PENDING\]\s*$/i) {
2219             $section = 'pending';
2220             if (defined($descr)) {
2221                 $descr =~ s/\s+$//;
2222                 $conf->{description} = $descr;
2223             }
2224             $descr = undef;
2225             $conf = $res->{$section} = {};
2226             next;
2227
2228         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2229             $section = $1;
2230             if (defined($descr)) {
2231                 $descr =~ s/\s+$//;
2232                 $conf->{description} = $descr;
2233             }
2234             $descr = undef;
2235             $conf = $res->{snapshots}->{$section} = {};
2236             next;
2237         }
2238
2239         if ($line =~ m/^\#(.*)\s*$/) {
2240             $descr = '' if !defined($descr);
2241             $descr .= PVE::Tools::decode_text($1) . "\n";
2242             next;
2243         }
2244
2245         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2246             $descr = '' if !defined($descr);
2247             $descr .= PVE::Tools::decode_text($2);
2248         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2249             $conf->{snapstate} = $1;
2250         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2251             my $key = $1;
2252             my $value = $2;
2253             $conf->{$key} = $value;
2254         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2255             my $value = $1;
2256             if ($section eq 'pending') {
2257                 $conf->{delete} = $value; # we parse this later
2258             } else {
2259                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2260             }
2261         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2262             my $key = $1;
2263             my $value = $2;
2264             eval { $value = check_type($key, $value); };
2265             if ($@) {
2266                 warn "vm $vmid - unable to parse value of '$key' - $@";
2267             } else {
2268                 $key = 'ide2' if $key eq 'cdrom';
2269                 my $fmt = $confdesc->{$key}->{format};
2270                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2271                     my $v = parse_drive($key, $value);
2272                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2273                         $v->{file} = $volid;
2274                         $value = print_drive($vmid, $v);
2275                     } else {
2276                         warn "vm $vmid - unable to parse value of '$key'\n";
2277                         next;
2278                     }
2279                 }
2280
2281                 $conf->{$key} = $value;
2282             }
2283         }
2284     }
2285
2286     if (defined($descr)) {
2287         $descr =~ s/\s+$//;
2288         $conf->{description} = $descr;
2289     }
2290     delete $res->{snapstate}; # just to be sure
2291
2292     return $res;
2293 }
2294
2295 sub write_vm_config {
2296     my ($filename, $conf) = @_;
2297
2298     delete $conf->{snapstate}; # just to be sure
2299
2300     if ($conf->{cdrom}) {
2301         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2302         $conf->{ide2} = $conf->{cdrom};
2303         delete $conf->{cdrom};
2304     }
2305
2306     # we do not use 'smp' any longer
2307     if ($conf->{sockets}) {
2308         delete $conf->{smp};
2309     } elsif ($conf->{smp}) {
2310         $conf->{sockets} = $conf->{smp};
2311         delete $conf->{cores};
2312         delete $conf->{smp};
2313     }
2314
2315     my $used_volids = {};
2316
2317     my $cleanup_config = sub {
2318         my ($cref, $pending, $snapname) = @_;
2319
2320         foreach my $key (keys %$cref) {
2321             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2322                 $key eq 'snapstate' || $key eq 'pending';
2323             my $value = $cref->{$key};
2324             if ($key eq 'delete') {
2325                 die "propertry 'delete' is only allowed in [PENDING]\n"
2326                     if !$pending;
2327                 # fixme: check syntax?
2328                 next;
2329             }
2330             eval { $value = check_type($key, $value); };
2331             die "unable to parse value of '$key' - $@" if $@;
2332
2333             $cref->{$key} = $value;
2334
2335             if (!$snapname && is_valid_drivename($key)) {
2336                 my $drive = parse_drive($key, $value);
2337                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2338             }
2339         }
2340     };
2341
2342     &$cleanup_config($conf);
2343
2344     &$cleanup_config($conf->{pending}, 1);
2345
2346     foreach my $snapname (keys %{$conf->{snapshots}}) {
2347         die "internal error" if $snapname eq 'pending';
2348         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2349     }
2350
2351     # remove 'unusedX' settings if we re-add a volume
2352     foreach my $key (keys %$conf) {
2353         my $value = $conf->{$key};
2354         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2355             delete $conf->{$key};
2356         }
2357     }
2358
2359     my $generate_raw_config = sub {
2360         my ($conf, $pending) = @_;
2361
2362         my $raw = '';
2363
2364         # add description as comment to top of file
2365         if (defined(my $descr = $conf->{description})) {
2366             if ($descr) {
2367                 foreach my $cl (split(/\n/, $descr)) {
2368                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2369                 }
2370             } else {
2371                 $raw .= "#\n" if $pending;
2372             }
2373         }
2374
2375         foreach my $key (sort keys %$conf) {
2376             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2377             $raw .= "$key: $conf->{$key}\n";
2378         }
2379         return $raw;
2380     };
2381
2382     my $raw = &$generate_raw_config($conf);
2383
2384     if (scalar(keys %{$conf->{pending}})){
2385         $raw .= "\n[PENDING]\n";
2386         $raw .= &$generate_raw_config($conf->{pending}, 1);
2387     }
2388
2389     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2390         $raw .= "\n[$snapname]\n";
2391         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2392     }
2393
2394     return $raw;
2395 }
2396
2397 sub load_defaults {
2398
2399     my $res = {};
2400
2401     # we use static defaults from our JSON schema configuration
2402     foreach my $key (keys %$confdesc) {
2403         if (defined(my $default = $confdesc->{$key}->{default})) {
2404             $res->{$key} = $default;
2405         }
2406     }
2407
2408     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2409     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2410
2411     return $res;
2412 }
2413
2414 sub config_list {
2415     my $vmlist = PVE::Cluster::get_vmlist();
2416     my $res = {};
2417     return $res if !$vmlist || !$vmlist->{ids};
2418     my $ids = $vmlist->{ids};
2419
2420     foreach my $vmid (keys %$ids) {
2421         my $d = $ids->{$vmid};
2422         next if !$d->{node} || $d->{node} ne $nodename;
2423         next if !$d->{type} || $d->{type} ne 'qemu';
2424         $res->{$vmid}->{exists} = 1;
2425     }
2426     return $res;
2427 }
2428
2429 # test if VM uses local resources (to prevent migration)
2430 sub check_local_resources {
2431     my ($conf, $noerr) = @_;
2432
2433     my $loc_res = 0;
2434
2435     $loc_res = 1 if $conf->{hostusb}; # old syntax
2436     $loc_res = 1 if $conf->{hostpci}; # old syntax
2437
2438     foreach my $k (keys %$conf) {
2439         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2440         # sockets are safe: they will recreated be on the target side post-migrate
2441         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2442         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2443     }
2444
2445     die "VM uses local resources\n" if $loc_res && !$noerr;
2446
2447     return $loc_res;
2448 }
2449
2450 # check if used storages are available on all nodes (use by migrate)
2451 sub check_storage_availability {
2452     my ($storecfg, $conf, $node) = @_;
2453
2454     foreach_drive($conf, sub {
2455         my ($ds, $drive) = @_;
2456
2457         my $volid = $drive->{file};
2458         return if !$volid;
2459
2460         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2461         return if !$sid;
2462
2463         # check if storage is available on both nodes
2464         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2465         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2466    });
2467 }
2468
2469 # list nodes where all VM images are available (used by has_feature API)
2470 sub shared_nodes {
2471     my ($conf, $storecfg) = @_;
2472
2473     my $nodelist = PVE::Cluster::get_nodelist();
2474     my $nodehash = { map { $_ => 1 } @$nodelist };
2475     my $nodename = PVE::INotify::nodename();
2476
2477     foreach_drive($conf, sub {
2478         my ($ds, $drive) = @_;
2479
2480         my $volid = $drive->{file};
2481         return if !$volid;
2482
2483         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2484         if ($storeid) {
2485             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2486             if ($scfg->{disable}) {
2487                 $nodehash = {};
2488             } elsif (my $avail = $scfg->{nodes}) {
2489                 foreach my $node (keys %$nodehash) {
2490                     delete $nodehash->{$node} if !$avail->{$node};
2491                 }
2492             } elsif (!$scfg->{shared}) {
2493                 foreach my $node (keys %$nodehash) {
2494                     delete $nodehash->{$node} if $node ne $nodename
2495                 }
2496             }
2497         }
2498     });
2499
2500     return $nodehash
2501 }
2502
2503 sub check_cmdline {
2504     my ($pidfile, $pid) = @_;
2505
2506     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2507     if (defined($fh)) {
2508         my $line = <$fh>;
2509         $fh->close;
2510         return undef if !$line;
2511         my @param = split(/\0/, $line);
2512
2513         my $cmd = $param[0];
2514         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2515
2516         for (my $i = 0; $i < scalar (@param); $i++) {
2517             my $p = $param[$i];
2518             next if !$p;
2519             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2520                 my $p = $param[$i+1];
2521                 return 1 if $p && ($p eq $pidfile);
2522                 return undef;
2523             }
2524         }
2525     }
2526     return undef;
2527 }
2528
2529 sub check_running {
2530     my ($vmid, $nocheck, $node) = @_;
2531
2532     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2533
2534     die "unable to find configuration file for VM $vmid - no such machine\n"
2535         if !$nocheck && ! -f $filename;
2536
2537     my $pidfile = pidfile_name($vmid);
2538
2539     if (my $fd = IO::File->new("<$pidfile")) {
2540         my $st = stat($fd);
2541         my $line = <$fd>;
2542         close($fd);
2543
2544         my $mtime = $st->mtime;
2545         if ($mtime > time()) {
2546             warn "file '$filename' modified in future\n";
2547         }
2548
2549         if ($line =~ m/^(\d+)$/) {
2550             my $pid = $1;
2551             if (check_cmdline($pidfile, $pid)) {
2552                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2553                     return $pid;
2554                 }
2555             }
2556         }
2557     }
2558
2559     return undef;
2560 }
2561
2562 sub vzlist {
2563
2564     my $vzlist = config_list();
2565
2566     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2567
2568     while (defined(my $de = $fd->read)) {
2569         next if $de !~ m/^(\d+)\.pid$/;
2570         my $vmid = $1;
2571         next if !defined($vzlist->{$vmid});
2572         if (my $pid = check_running($vmid)) {
2573             $vzlist->{$vmid}->{pid} = $pid;
2574         }
2575     }
2576
2577     return $vzlist;
2578 }
2579
2580 sub disksize {
2581     my ($storecfg, $conf) = @_;
2582
2583     my $bootdisk = $conf->{bootdisk};
2584     return undef if !$bootdisk;
2585     return undef if !is_valid_drivename($bootdisk);
2586
2587     return undef if !$conf->{$bootdisk};
2588
2589     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2590     return undef if !defined($drive);
2591
2592     return undef if drive_is_cdrom($drive);
2593
2594     my $volid = $drive->{file};
2595     return undef if !$volid;
2596
2597     return $drive->{size};
2598 }
2599
2600 my $last_proc_pid_stat;
2601
2602 # get VM status information
2603 # This must be fast and should not block ($full == false)
2604 # We only query KVM using QMP if $full == true (this can be slow)
2605 sub vmstatus {
2606     my ($opt_vmid, $full) = @_;
2607
2608     my $res = {};
2609
2610     my $storecfg = PVE::Storage::config();
2611
2612     my $list = vzlist();
2613     my $defaults = load_defaults();
2614
2615     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2616
2617     my $cpucount = $cpuinfo->{cpus} || 1;
2618
2619     foreach my $vmid (keys %$list) {
2620         next if $opt_vmid && ($vmid ne $opt_vmid);
2621
2622         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2623         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2624
2625         my $d = {};
2626         $d->{pid} = $list->{$vmid}->{pid};
2627
2628         # fixme: better status?
2629         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2630
2631         my $size = disksize($storecfg, $conf);
2632         if (defined($size)) {
2633             $d->{disk} = 0; # no info available
2634             $d->{maxdisk} = $size;
2635         } else {
2636             $d->{disk} = 0;
2637             $d->{maxdisk} = 0;
2638         }
2639
2640         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2641             * ($conf->{cores} || $defaults->{cores});
2642         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2643         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2644
2645         $d->{name} = $conf->{name} || "VM $vmid";
2646         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2647             : $defaults->{memory}*(1024*1024);
2648
2649         if ($conf->{balloon}) {
2650             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2651             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2652                 : $defaults->{shares};
2653         }
2654
2655         $d->{uptime} = 0;
2656         $d->{cpu} = 0;
2657         $d->{mem} = 0;
2658
2659         $d->{netout} = 0;
2660         $d->{netin} = 0;
2661
2662         $d->{diskread} = 0;
2663         $d->{diskwrite} = 0;
2664
2665         $d->{template} = PVE::QemuConfig->is_template($conf);
2666
2667         $res->{$vmid} = $d;
2668     }
2669
2670     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2671     foreach my $dev (keys %$netdev) {
2672         next if $dev !~ m/^tap([1-9]\d*)i/;
2673         my $vmid = $1;
2674         my $d = $res->{$vmid};
2675         next if !$d;
2676
2677         $d->{netout} += $netdev->{$dev}->{receive};
2678         $d->{netin} += $netdev->{$dev}->{transmit};
2679
2680         if ($full) {
2681             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2682             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2683         }
2684
2685     }
2686
2687     my $ctime = gettimeofday;
2688
2689     foreach my $vmid (keys %$list) {
2690
2691         my $d = $res->{$vmid};
2692         my $pid = $d->{pid};
2693         next if !$pid;
2694
2695         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2696         next if !$pstat; # not running
2697
2698         my $used = $pstat->{utime} + $pstat->{stime};
2699
2700         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2701
2702         if ($pstat->{vsize}) {
2703             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2704         }
2705
2706         my $old = $last_proc_pid_stat->{$pid};
2707         if (!$old) {
2708             $last_proc_pid_stat->{$pid} = {
2709                 time => $ctime,
2710                 used => $used,
2711                 cpu => 0,
2712             };
2713             next;
2714         }
2715
2716         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2717
2718         if ($dtime > 1000) {
2719             my $dutime = $used -  $old->{used};
2720
2721             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2722             $last_proc_pid_stat->{$pid} = {
2723                 time => $ctime,
2724                 used => $used,
2725                 cpu => $d->{cpu},
2726             };
2727         } else {
2728             $d->{cpu} = $old->{cpu};
2729         }
2730     }
2731
2732     return $res if !$full;
2733
2734     my $qmpclient = PVE::QMPClient->new();
2735
2736     my $ballooncb = sub {
2737         my ($vmid, $resp) = @_;
2738
2739         my $info = $resp->{'return'};
2740         return if !$info->{max_mem};
2741
2742         my $d = $res->{$vmid};
2743
2744         # use memory assigned to VM
2745         $d->{maxmem} = $info->{max_mem};
2746         $d->{balloon} = $info->{actual};
2747
2748         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2749             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2750             $d->{freemem} = $info->{free_mem};
2751         }
2752
2753         $d->{ballooninfo} = $info;
2754     };
2755
2756     my $blockstatscb = sub {
2757         my ($vmid, $resp) = @_;
2758         my $data = $resp->{'return'} || [];
2759         my $totalrdbytes = 0;
2760         my $totalwrbytes = 0;
2761
2762         for my $blockstat (@$data) {
2763             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2764             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2765
2766             $blockstat->{device} =~ s/drive-//;
2767             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2768         }
2769         $res->{$vmid}->{diskread} = $totalrdbytes;
2770         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2771     };
2772
2773     my $statuscb = sub {
2774         my ($vmid, $resp) = @_;
2775
2776         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2777         # this fails if ballon driver is not loaded, so this must be
2778         # the last commnand (following command are aborted if this fails).
2779         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2780
2781         my $status = 'unknown';
2782         if (!defined($status = $resp->{'return'}->{status})) {
2783             warn "unable to get VM status\n";
2784             return;
2785         }
2786
2787         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2788     };
2789
2790     foreach my $vmid (keys %$list) {
2791         next if $opt_vmid && ($vmid ne $opt_vmid);
2792         next if !$res->{$vmid}->{pid}; # not running
2793         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2794     }
2795
2796     $qmpclient->queue_execute(undef, 2);
2797
2798     foreach my $vmid (keys %$list) {
2799         next if $opt_vmid && ($vmid ne $opt_vmid);
2800         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2801     }
2802
2803     return $res;
2804 }
2805
2806 sub foreach_drive {
2807     my ($conf, $func, @param) = @_;
2808
2809     foreach my $ds (valid_drive_names()) {
2810         next if !defined($conf->{$ds});
2811
2812         my $drive = parse_drive($ds, $conf->{$ds});
2813         next if !$drive;
2814
2815         &$func($ds, $drive, @param);
2816     }
2817 }
2818
2819 sub foreach_volid {
2820     my ($conf, $func, @param) = @_;
2821
2822     my $volhash = {};
2823
2824     my $test_volid = sub {
2825         my ($volid, $is_cdrom, $replicate, $snapname) = @_;
2826
2827         return if !$volid;
2828
2829         $volhash->{$volid}->{cdrom} //= 1;
2830         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2831
2832         $volhash->{$volid}->{replicate} //= 0;
2833         $volhash->{$volid}->{replicate} = 1 if $replicate;
2834
2835         $volhash->{$volid}->{referenced_in_config} //= 0;
2836         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
2837
2838         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
2839             if defined($snapname);
2840     };
2841
2842     foreach_drive($conf, sub {
2843         my ($ds, $drive) = @_;
2844         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, undef);
2845     });
2846
2847     foreach my $snapname (keys %{$conf->{snapshots}}) {
2848         my $snap = $conf->{snapshots}->{$snapname};
2849         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
2850         foreach_drive($snap, sub {
2851             my ($ds, $drive) = @_;
2852             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $snapname);
2853         });
2854     }
2855
2856     foreach my $volid (keys %$volhash) {
2857         &$func($volid, $volhash->{$volid}, @param);
2858     }
2859 }
2860
2861 sub vga_conf_has_spice {
2862     my ($vga) = @_;
2863
2864     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2865
2866     return $1 || 1;
2867 }
2868
2869 sub config_to_command {
2870     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2871
2872     my $cmd = [];
2873     my $globalFlags = [];
2874     my $machineFlags = [];
2875     my $rtcFlags = [];
2876     my $cpuFlags = [];
2877     my $devices = [];
2878     my $pciaddr = '';
2879     my $bridges = {};
2880     my $kvmver = kvm_user_version();
2881     my $vernum = 0; # unknown
2882     my $ostype = $conf->{ostype};
2883     my $winversion = windows_version($ostype);
2884     my $kvm = $conf->{kvm} // 1;
2885
2886     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
2887
2888     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2889         $vernum = $1*1000000+$2*1000;
2890     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2891         $vernum = $1*1000000+$2*1000+$3;
2892     }
2893
2894     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2895
2896     my $have_ovz = -f '/proc/vz/vestat';
2897
2898     my $q35 = machine_type_is_q35($conf);
2899     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2900     my $machine_type = $forcemachine || $conf->{machine};
2901     my $use_old_bios_files = undef;
2902     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2903
2904     my $cpuunits = defined($conf->{cpuunits}) ?
2905             $conf->{cpuunits} : $defaults->{cpuunits};
2906
2907     push @$cmd, '/usr/bin/kvm';
2908
2909     push @$cmd, '-id', $vmid;
2910
2911     my $use_virtio = 0;
2912
2913     my $qmpsocket = qmp_socket($vmid);
2914     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2915     push @$cmd, '-mon', "chardev=qmp,mode=control";
2916
2917
2918     push @$cmd, '-pidfile' , pidfile_name($vmid);
2919
2920     push @$cmd, '-daemonize';
2921
2922     if ($conf->{smbios1}) {
2923         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2924     }
2925
2926     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2927         die "uefi base image not found\n" if ! -f $OVMF_CODE;
2928
2929         my $path;
2930         my $format;
2931         if (my $efidisk = $conf->{efidisk0}) {
2932             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
2933             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2934             $format = $d->{format};
2935             if ($storeid) {
2936                 $path = PVE::Storage::path($storecfg, $d->{file});
2937                 if (!defined($format)) {
2938                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2939                     $format = qemu_img_format($scfg, $volname);
2940                 }
2941             } else {
2942                 $path = $d->{file};
2943                 die "efidisk format must be specified\n"
2944                     if !defined($format);
2945             }
2946         } else {
2947             warn "no efidisk configured! Using temporary efivars disk.\n";
2948             $path = "/tmp/$vmid-ovmf.fd";
2949             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
2950             $format = 'raw';
2951         }
2952
2953         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
2954         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
2955     }
2956
2957
2958     # add usb controllers
2959     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2960     push @$devices, @usbcontrollers if @usbcontrollers;
2961     my $vga = $conf->{vga};
2962
2963     my $qxlnum = vga_conf_has_spice($vga);
2964     $vga = 'qxl' if $qxlnum;
2965
2966     if (!$vga) {
2967         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
2968             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
2969         } else {
2970             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
2971         }
2972     }
2973
2974     # enable absolute mouse coordinates (needed by vnc)
2975     my $tablet;
2976     if (defined($conf->{tablet})) {
2977         $tablet = $conf->{tablet};
2978     } else {
2979         $tablet = $defaults->{tablet};
2980         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2981         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2982     }
2983
2984     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2985
2986     my $kvm_off = 0;
2987     my $gpu_passthrough;
2988
2989     # host pci devices
2990     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2991         my $d = parse_hostpci($conf->{"hostpci$i"});
2992         next if !$d;
2993
2994         my $pcie = $d->{pcie};
2995         if($pcie){
2996             die "q35 machine model is not enabled" if !$q35;
2997             $pciaddr = print_pcie_addr("hostpci$i");
2998         }else{
2999             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3000         }
3001
3002         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3003         my $romfile = $d->{romfile};
3004
3005         my $xvga = '';
3006         if ($d->{'x-vga'}) {
3007             $xvga = ',x-vga=on';
3008             $kvm_off = 1;
3009             $vga = 'none';
3010             $gpu_passthrough = 1;
3011
3012             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3013                 $xvga = "";
3014             }
3015         }
3016         my $pcidevices = $d->{pciid};
3017         my $multifunction = 1 if @$pcidevices > 1;
3018
3019         my $j=0;
3020         foreach my $pcidevice (@$pcidevices) {
3021
3022             my $id = "hostpci$i";
3023             $id .= ".$j" if $multifunction;
3024             my $addr = $pciaddr;
3025             $addr .= ".$j" if $multifunction;
3026             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3027
3028             if($j == 0){
3029                 $devicestr .= "$rombar$xvga";
3030                 $devicestr .= ",multifunction=on" if $multifunction;
3031                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3032             }
3033
3034             push @$devices, '-device', $devicestr;
3035             $j++;
3036         }
3037     }
3038
3039     # usb devices
3040     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3041     push @$devices, @usbdevices if @usbdevices;
3042     # serial devices
3043     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3044         if (my $path = $conf->{"serial$i"}) {
3045             if ($path eq 'socket') {
3046                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3047                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3048                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3049             } else {
3050                 die "no such serial device\n" if ! -c $path;
3051                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3052                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3053             }
3054         }
3055     }
3056
3057     # parallel devices
3058     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3059         if (my $path = $conf->{"parallel$i"}) {
3060             die "no such parallel device\n" if ! -c $path;
3061             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3062             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3063             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3064         }
3065     }
3066
3067     my $vmname = $conf->{name} || "vm$vmid";
3068
3069     push @$cmd, '-name', $vmname;
3070
3071     my $sockets = 1;
3072     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3073     $sockets = $conf->{sockets} if  $conf->{sockets};
3074
3075     my $cores = $conf->{cores} || 1;
3076
3077     my $maxcpus = $sockets * $cores;
3078
3079     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3080
3081     my $allowed_vcpus = $cpuinfo->{cpus};
3082
3083     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3084         if ($allowed_vcpus < $maxcpus);
3085
3086     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3087
3088         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3089         for (my $i = 2; $i <= $vcpus; $i++)  {
3090             my $cpustr = print_cpu_device($conf,$i);
3091             push @$cmd, '-device', $cpustr;
3092         }
3093
3094     } else {
3095
3096         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3097     }
3098     push @$cmd, '-nodefaults';
3099
3100     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3101
3102     my $bootindex_hash = {};
3103     my $i = 1;
3104     foreach my $o (split(//, $bootorder)) {
3105         $bootindex_hash->{$o} = $i*100;
3106         $i++;
3107     }
3108
3109     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3110
3111     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3112
3113     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3114
3115     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3116
3117     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3118         my $socket = vnc_socket($vmid);
3119         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3120     } else {
3121         push @$cmd, '-nographic';
3122     }
3123
3124     # time drift fix
3125     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3126
3127     my $useLocaltime = $conf->{localtime};
3128
3129     if ($winversion >= 5) { # windows
3130         $useLocaltime = 1 if !defined($conf->{localtime});
3131
3132         # use time drift fix when acpi is enabled
3133         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3134             $tdf = 1 if !defined($conf->{tdf});
3135         }
3136     }
3137
3138     if ($winversion >= 6) {
3139         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3140         push @$cmd, '-no-hpet';
3141     }
3142
3143     push @$rtcFlags, 'driftfix=slew' if $tdf;
3144
3145     if (!$kvm) {
3146         push @$machineFlags, 'accel=tcg';
3147     }
3148
3149     if ($machine_type) {
3150         push @$machineFlags, "type=${machine_type}";
3151     }
3152
3153     if ($conf->{startdate}) {
3154         push @$rtcFlags, "base=$conf->{startdate}";
3155     } elsif ($useLocaltime) {
3156         push @$rtcFlags, 'base=localtime';
3157     }
3158
3159     my $cpu = $kvm ? "kvm64" : "qemu64";
3160     if (my $cputype = $conf->{cpu}) {
3161         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3162             or die "Cannot parse cpu description: $cputype\n";
3163         $cpu = $cpuconf->{cputype};
3164         $kvm_off = 1 if $cpuconf->{hidden};
3165
3166         if (defined(my $flags = $cpuconf->{flags})) {
3167             push @$cpuFlags, $flags;
3168         }
3169     }
3170
3171     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3172
3173     push @$cpuFlags , '-x2apic'
3174         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3175
3176     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3177
3178     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3179
3180     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3181
3182         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3183         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3184     }
3185
3186     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3187
3188     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3189
3190     push @$cpuFlags, 'kvm=off' if $kvm_off;
3191
3192     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3193         die "internal error"; # should not happen
3194
3195     push @$cpuFlags, "vendor=${cpu_vendor}"
3196         if $cpu_vendor ne 'default';
3197
3198     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3199
3200     push @$cmd, '-cpu', $cpu;
3201
3202     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3203
3204     push @$cmd, '-S' if $conf->{freeze};
3205
3206     # set keyboard layout
3207     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3208     push @$cmd, '-k', $kb if $kb;
3209
3210     # enable sound
3211     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3212     #push @$cmd, '-soundhw', 'es1370';
3213     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3214
3215     if($conf->{agent}) {
3216         my $qgasocket = qmp_socket($vmid, 1);
3217         my $pciaddr = print_pci_addr("qga0", $bridges);
3218         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3219         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3220         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3221     }
3222
3223     my $spice_port;
3224
3225     if ($qxlnum) {
3226         if ($qxlnum > 1) {
3227             if ($winversion){
3228                 for(my $i = 1; $i < $qxlnum; $i++){
3229                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3230                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3231                 }
3232             } else {
3233                 # assume other OS works like Linux
3234                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3235                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3236             }
3237         }
3238
3239         my $pciaddr = print_pci_addr("spice", $bridges);
3240
3241         my $nodename = PVE::INotify::nodename();
3242         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3243         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3244         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3245         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3246         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3247
3248         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3249
3250         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3251         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3252         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3253     }
3254
3255     # enable balloon by default, unless explicitly disabled
3256     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3257         $pciaddr = print_pci_addr("balloon0", $bridges);
3258         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3259     }
3260
3261     if ($conf->{watchdog}) {
3262         my $wdopts = parse_watchdog($conf->{watchdog});
3263         $pciaddr = print_pci_addr("watchdog", $bridges);
3264         my $watchdog = $wdopts->{model} || 'i6300esb';
3265         push @$devices, '-device', "$watchdog$pciaddr";
3266         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3267     }
3268
3269     my $vollist = [];
3270     my $scsicontroller = {};
3271     my $ahcicontroller = {};
3272     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3273
3274     # Add iscsi initiator name if available
3275     if (my $initiator = get_initiator_name()) {
3276         push @$devices, '-iscsi', "initiator-name=$initiator";
3277     }
3278
3279     foreach_drive($conf, sub {
3280         my ($ds, $drive) = @_;
3281
3282         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3283             push @$vollist, $drive->{file};
3284         }
3285
3286         # ignore efidisk here, already added in bios/fw handling code above
3287         return if $drive->{interface} eq 'efidisk';
3288
3289         $use_virtio = 1 if $ds =~ m/^virtio/;
3290
3291         if (drive_is_cdrom ($drive)) {
3292             if ($bootindex_hash->{d}) {
3293                 $drive->{bootindex} = $bootindex_hash->{d};
3294                 $bootindex_hash->{d} += 1;
3295             }
3296         } else {
3297             if ($bootindex_hash->{c}) {
3298                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3299                 $bootindex_hash->{c} += 1;
3300             }
3301         }
3302
3303         if($drive->{interface} eq 'virtio'){
3304            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3305         }
3306
3307         if ($drive->{interface} eq 'scsi') {
3308
3309             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3310
3311             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3312             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3313
3314             my $iothread = '';
3315             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3316                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3317                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3318             } elsif ($drive->{iothread}) {
3319                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3320             }
3321
3322             my $queues = '';
3323             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3324                 $queues = ",num_queues=$drive->{queues}";
3325             }
3326
3327             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3328             $scsicontroller->{$controller}=1;
3329         }
3330
3331         if ($drive->{interface} eq 'sata') {
3332            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3333            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3334            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3335            $ahcicontroller->{$controller}=1;
3336         }
3337
3338         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3339         push @$devices, '-drive',$drive_cmd;
3340         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3341     });
3342
3343     for (my $i = 0; $i < $MAX_NETS; $i++) {
3344          next if !$conf->{"net$i"};
3345          my $d = parse_net($conf->{"net$i"});
3346          next if !$d;
3347
3348          $use_virtio = 1 if $d->{model} eq 'virtio';
3349
3350          if ($bootindex_hash->{n}) {
3351             $d->{bootindex} = $bootindex_hash->{n};
3352             $bootindex_hash->{n} += 1;
3353          }
3354
3355          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3356          push @$devices, '-netdev', $netdevfull;
3357
3358          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3359          push @$devices, '-device', $netdevicefull;
3360     }
3361
3362     if (!$q35) {
3363         # add pci bridges
3364         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3365            $bridges->{1} = 1;
3366            $bridges->{2} = 1;
3367         }
3368
3369         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3370
3371         while (my ($k, $v) = each %$bridges) {
3372             $pciaddr = print_pci_addr("pci.$k");
3373             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3374         }
3375     }
3376
3377     # add custom args
3378     if ($conf->{args}) {
3379         my $aa = PVE::Tools::split_args($conf->{args});
3380         push @$cmd, @$aa;
3381     }
3382
3383     push @$cmd, @$devices;
3384     push @$cmd, '-rtc', join(',', @$rtcFlags)
3385         if scalar(@$rtcFlags);
3386     push @$cmd, '-machine', join(',', @$machineFlags)
3387         if scalar(@$machineFlags);
3388     push @$cmd, '-global', join(',', @$globalFlags)
3389         if scalar(@$globalFlags);
3390
3391     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3392 }
3393
3394 sub vnc_socket {
3395     my ($vmid) = @_;
3396     return "${var_run_tmpdir}/$vmid.vnc";
3397 }
3398
3399 sub spice_port {
3400     my ($vmid) = @_;
3401
3402     my $res = vm_mon_cmd($vmid, 'query-spice');
3403
3404     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3405 }
3406
3407 sub qmp_socket {
3408     my ($vmid, $qga) = @_;
3409     my $sockettype = $qga ? 'qga' : 'qmp';
3410     return "${var_run_tmpdir}/$vmid.$sockettype";
3411 }
3412
3413 sub pidfile_name {
3414     my ($vmid) = @_;
3415     return "${var_run_tmpdir}/$vmid.pid";
3416 }
3417
3418 sub vm_devices_list {
3419     my ($vmid) = @_;
3420
3421     my $res = vm_mon_cmd($vmid, 'query-pci');
3422     my $devices = {};
3423     foreach my $pcibus (@$res) {
3424         foreach my $device (@{$pcibus->{devices}}) {
3425             next if !$device->{'qdev_id'};
3426             if ($device->{'pci_bridge'}) {
3427                 $devices->{$device->{'qdev_id'}} = 1;
3428                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3429                     next if !$bridge_device->{'qdev_id'};
3430                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3431                     $devices->{$device->{'qdev_id'}}++;
3432                 }
3433             } else {
3434                 $devices->{$device->{'qdev_id'}} = 1;
3435             }
3436         }
3437     }
3438
3439     my $resblock = vm_mon_cmd($vmid, 'query-block');
3440     foreach my $block (@$resblock) {
3441         if($block->{device} =~ m/^drive-(\S+)/){
3442                 $devices->{$1} = 1;
3443         }
3444     }
3445
3446     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3447     foreach my $mice (@$resmice) {
3448         if ($mice->{name} eq 'QEMU HID Tablet') {
3449             $devices->{tablet} = 1;
3450             last;
3451         }
3452     }
3453
3454     # for usb devices there is no query-usb
3455     # but we can iterate over the entries in
3456     # qom-list path=/machine/peripheral
3457     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3458     foreach my $per (@$resperipheral) {
3459         if ($per->{name} =~ m/^usb\d+$/) {
3460             $devices->{$per->{name}} = 1;
3461         }
3462     }
3463
3464     return $devices;
3465 }
3466
3467 sub vm_deviceplug {
3468     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3469
3470     my $q35 = machine_type_is_q35($conf);
3471
3472     my $devices_list = vm_devices_list($vmid);
3473     return 1 if defined($devices_list->{$deviceid});
3474
3475     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3476
3477     if ($deviceid eq 'tablet') {
3478
3479         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3480
3481     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3482
3483         die "usb hotplug currently not reliable\n";
3484         # since we can't reliably hot unplug all added usb devices
3485         # and usb passthrough disables live migration
3486         # we disable usb hotplugging for now
3487         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3488
3489     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3490
3491         qemu_iothread_add($vmid, $deviceid, $device);
3492
3493         qemu_driveadd($storecfg, $vmid, $device);
3494         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3495
3496         qemu_deviceadd($vmid, $devicefull);
3497         eval { qemu_deviceaddverify($vmid, $deviceid); };
3498         if (my $err = $@) {
3499             eval { qemu_drivedel($vmid, $deviceid); };
3500             warn $@ if $@;
3501             die $err;
3502         }
3503
3504     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3505
3506
3507         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3508         my $pciaddr = print_pci_addr($deviceid);
3509         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3510
3511         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3512
3513         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3514             qemu_iothread_add($vmid, $deviceid, $device);
3515             $devicefull .= ",iothread=iothread-$deviceid";
3516         }
3517
3518         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3519             $devicefull .= ",num_queues=$device->{queues}";
3520         }
3521
3522         qemu_deviceadd($vmid, $devicefull);
3523         qemu_deviceaddverify($vmid, $deviceid);
3524
3525     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3526
3527         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3528         qemu_driveadd($storecfg, $vmid, $device);
3529
3530         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3531         eval { qemu_deviceadd($vmid, $devicefull); };
3532         if (my $err = $@) {
3533             eval { qemu_drivedel($vmid, $deviceid); };
3534             warn $@ if $@;
3535             die $err;
3536         }
3537
3538     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3539
3540         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3541
3542         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3543         my $use_old_bios_files = undef;
3544         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3545
3546         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3547         qemu_deviceadd($vmid, $netdevicefull);
3548         eval { qemu_deviceaddverify($vmid, $deviceid); };
3549         if (my $err = $@) {
3550             eval { qemu_netdevdel($vmid, $deviceid); };
3551             warn $@ if $@;
3552             die $err;
3553         }
3554
3555     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3556
3557         my $bridgeid = $2;
3558         my $pciaddr = print_pci_addr($deviceid);
3559         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3560
3561         qemu_deviceadd($vmid, $devicefull);
3562         qemu_deviceaddverify($vmid, $deviceid);
3563
3564     } else {
3565         die "can't hotplug device '$deviceid'\n";
3566     }
3567
3568     return 1;
3569 }
3570
3571 # fixme: this should raise exceptions on error!
3572 sub vm_deviceunplug {
3573     my ($vmid, $conf, $deviceid) = @_;
3574
3575     my $devices_list = vm_devices_list($vmid);
3576     return 1 if !defined($devices_list->{$deviceid});
3577
3578     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3579
3580     if ($deviceid eq 'tablet') {
3581
3582         qemu_devicedel($vmid, $deviceid);
3583
3584     } elsif ($deviceid =~ m/^usb\d+$/) {
3585
3586         die "usb hotplug currently not reliable\n";
3587         # when unplugging usb devices this way,
3588         # there may be remaining usb controllers/hubs
3589         # so we disable it for now
3590         qemu_devicedel($vmid, $deviceid);
3591         qemu_devicedelverify($vmid, $deviceid);
3592
3593     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3594
3595         qemu_devicedel($vmid, $deviceid);
3596         qemu_devicedelverify($vmid, $deviceid);
3597         qemu_drivedel($vmid, $deviceid);
3598         qemu_iothread_del($conf, $vmid, $deviceid);
3599
3600     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3601
3602         qemu_devicedel($vmid, $deviceid);
3603         qemu_devicedelverify($vmid, $deviceid);
3604         qemu_iothread_del($conf, $vmid, $deviceid);
3605
3606     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3607
3608         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3609         my $device = parse_drive($deviceid, $conf->{$deviceid});
3610         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3611
3612         qemu_devicedel($vmid, $deviceid);
3613         qemu_drivedel($vmid, $deviceid);
3614         qemu_deletescsihw($conf, $vmid, $deviceid);
3615
3616     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3617
3618         qemu_devicedel($vmid, $deviceid);
3619         qemu_devicedelverify($vmid, $deviceid);
3620         qemu_netdevdel($vmid, $deviceid);
3621
3622     } else {
3623         die "can't unplug device '$deviceid'\n";
3624     }
3625
3626     return 1;
3627 }
3628
3629 sub qemu_deviceadd {
3630     my ($vmid, $devicefull) = @_;
3631
3632     $devicefull = "driver=".$devicefull;
3633     my %options =  split(/[=,]/, $devicefull);
3634
3635     vm_mon_cmd($vmid, "device_add" , %options);
3636 }
3637
3638 sub qemu_devicedel {
3639     my ($vmid, $deviceid) = @_;
3640
3641     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3642 }
3643
3644 sub qemu_iothread_add {
3645     my($vmid, $deviceid, $device) = @_;
3646
3647     if ($device->{iothread}) {
3648         my $iothreads = vm_iothreads_list($vmid);
3649         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3650     }
3651 }
3652
3653 sub qemu_iothread_del {
3654     my($conf, $vmid, $deviceid) = @_;
3655
3656     my $device = parse_drive($deviceid, $conf->{$deviceid});
3657     if ($device->{iothread}) {
3658         my $iothreads = vm_iothreads_list($vmid);
3659         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3660     }
3661 }
3662
3663 sub qemu_objectadd {
3664     my($vmid, $objectid, $qomtype) = @_;
3665
3666     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3667
3668     return 1;
3669 }
3670
3671 sub qemu_objectdel {
3672     my($vmid, $objectid) = @_;
3673
3674     vm_mon_cmd($vmid, "object-del", id => $objectid);
3675
3676     return 1;
3677 }
3678
3679 sub qemu_driveadd {
3680     my ($storecfg, $vmid, $device) = @_;
3681
3682     my $drive = print_drive_full($storecfg, $vmid, $device);
3683     $drive =~ s/\\/\\\\/g;
3684     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3685
3686     # If the command succeeds qemu prints: "OK"
3687     return 1 if $ret =~ m/OK/s;
3688
3689     die "adding drive failed: $ret\n";
3690 }
3691
3692 sub qemu_drivedel {
3693     my($vmid, $deviceid) = @_;
3694
3695     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3696     $ret =~ s/^\s+//;
3697
3698     return 1 if $ret eq "";
3699
3700     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3701     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3702
3703     die "deleting drive $deviceid failed : $ret\n";
3704 }
3705
3706 sub qemu_deviceaddverify {
3707     my ($vmid, $deviceid) = @_;
3708
3709     for (my $i = 0; $i <= 5; $i++) {
3710          my $devices_list = vm_devices_list($vmid);
3711          return 1 if defined($devices_list->{$deviceid});
3712          sleep 1;
3713     }
3714
3715     die "error on hotplug device '$deviceid'\n";
3716 }
3717
3718
3719 sub qemu_devicedelverify {
3720     my ($vmid, $deviceid) = @_;
3721
3722     # need to verify that the device is correctly removed as device_del
3723     # is async and empty return is not reliable
3724
3725     for (my $i = 0; $i <= 5; $i++) {
3726          my $devices_list = vm_devices_list($vmid);
3727          return 1 if !defined($devices_list->{$deviceid});
3728          sleep 1;
3729     }
3730
3731     die "error on hot-unplugging device '$deviceid'\n";
3732 }
3733
3734 sub qemu_findorcreatescsihw {
3735     my ($storecfg, $conf, $vmid, $device) = @_;
3736
3737     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3738
3739     my $scsihwid="$controller_prefix$controller";
3740     my $devices_list = vm_devices_list($vmid);
3741
3742     if(!defined($devices_list->{$scsihwid})) {
3743         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3744     }
3745
3746     return 1;
3747 }
3748
3749 sub qemu_deletescsihw {
3750     my ($conf, $vmid, $opt) = @_;
3751
3752     my $device = parse_drive($opt, $conf->{$opt});
3753
3754     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3755         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3756         return 1;
3757     }
3758
3759     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3760
3761     my $devices_list = vm_devices_list($vmid);
3762     foreach my $opt (keys %{$devices_list}) {
3763         if (PVE::QemuServer::is_valid_drivename($opt)) {
3764             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3765             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3766                 return 1;
3767             }
3768         }
3769     }
3770
3771     my $scsihwid="scsihw$controller";
3772
3773     vm_deviceunplug($vmid, $conf, $scsihwid);
3774
3775     return 1;
3776 }
3777
3778 sub qemu_add_pci_bridge {
3779     my ($storecfg, $conf, $vmid, $device) = @_;
3780
3781     my $bridges = {};
3782
3783     my $bridgeid;
3784
3785     print_pci_addr($device, $bridges);
3786
3787     while (my ($k, $v) = each %$bridges) {
3788         $bridgeid = $k;
3789     }
3790     return 1 if !defined($bridgeid) || $bridgeid < 1;
3791
3792     my $bridge = "pci.$bridgeid";
3793     my $devices_list = vm_devices_list($vmid);
3794
3795     if (!defined($devices_list->{$bridge})) {
3796         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3797     }
3798
3799     return 1;
3800 }
3801
3802 sub qemu_set_link_status {
3803     my ($vmid, $device, $up) = @_;
3804
3805     vm_mon_cmd($vmid, "set_link", name => $device,
3806                up => $up ? JSON::true : JSON::false);
3807 }
3808
3809 sub qemu_netdevadd {
3810     my ($vmid, $conf, $device, $deviceid) = @_;
3811
3812     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3813     my %options =  split(/[=,]/, $netdev);
3814
3815     vm_mon_cmd($vmid, "netdev_add",  %options);
3816     return 1;
3817 }
3818
3819 sub qemu_netdevdel {
3820     my ($vmid, $deviceid) = @_;
3821
3822     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3823 }
3824
3825 sub qemu_usb_hotplug {
3826     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3827
3828     return if !$device;
3829
3830     # remove the old one first
3831     vm_deviceunplug($vmid, $conf, $deviceid);
3832
3833     # check if xhci controller is necessary and available
3834     if ($device->{usb3}) {
3835
3836         my $devicelist = vm_devices_list($vmid);
3837
3838         if (!$devicelist->{xhci}) {
3839             my $pciaddr = print_pci_addr("xhci");
3840             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3841         }
3842     }
3843     my $d = parse_usb_device($device->{host});
3844     $d->{usb3} = $device->{usb3};
3845
3846     # add the new one
3847     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3848 }
3849
3850 sub qemu_cpu_hotplug {
3851     my ($vmid, $conf, $vcpus) = @_;
3852
3853     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3854
3855     my $sockets = 1;
3856     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3857     $sockets = $conf->{sockets} if  $conf->{sockets};
3858     my $cores = $conf->{cores} || 1;
3859     my $maxcpus = $sockets * $cores;
3860
3861     $vcpus = $maxcpus if !$vcpus;
3862
3863     die "you can't add more vcpus than maxcpus\n"
3864         if $vcpus > $maxcpus;
3865
3866     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3867
3868     if ($vcpus < $currentvcpus) {
3869
3870         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3871
3872             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3873                 qemu_devicedel($vmid, "cpu$i");
3874                 my $retry = 0;
3875                 my $currentrunningvcpus = undef;
3876                 while (1) {
3877                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3878                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3879                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3880                     $retry++;
3881                     sleep 1;
3882                 }
3883                 #update conf after each succesfull cpu unplug
3884                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3885                 PVE::QemuConfig->write_config($vmid, $conf);
3886             }
3887         } else {
3888             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3889         }
3890
3891         return;
3892     }
3893
3894     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3895     die "vcpus in running vm does not match its configuration\n"
3896         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3897
3898     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3899
3900         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3901             my $cpustr = print_cpu_device($conf, $i);
3902             qemu_deviceadd($vmid, $cpustr);
3903
3904             my $retry = 0;
3905             my $currentrunningvcpus = undef;
3906             while (1) {
3907                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3908                 last if scalar(@{$currentrunningvcpus}) == $i;
3909                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3910                 sleep 1;
3911                 $retry++;
3912             }
3913             #update conf after each succesfull cpu hotplug
3914             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3915             PVE::QemuConfig->write_config($vmid, $conf);
3916         }
3917     } else {
3918
3919         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3920             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3921         }
3922     }
3923 }
3924
3925 sub qemu_block_set_io_throttle {
3926     my ($vmid, $deviceid,
3927         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3928         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3929         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3930         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3931
3932     return if !check_running($vmid) ;
3933
3934     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3935         bps => int($bps),
3936         bps_rd => int($bps_rd),
3937         bps_wr => int($bps_wr),
3938         iops => int($iops),
3939         iops_rd => int($iops_rd),
3940         iops_wr => int($iops_wr),
3941         bps_max => int($bps_max),
3942         bps_rd_max => int($bps_rd_max),
3943         bps_wr_max => int($bps_wr_max),
3944         iops_max => int($iops_max),
3945         iops_rd_max => int($iops_rd_max),
3946         iops_wr_max => int($iops_wr_max),
3947         bps_max_length => int($bps_max_length),
3948         bps_rd_max_length => int($bps_rd_max_length),
3949         bps_wr_max_length => int($bps_wr_max_length),
3950         iops_max_length => int($iops_max_length),
3951         iops_rd_max_length => int($iops_rd_max_length),
3952         iops_wr_max_length => int($iops_wr_max_length),
3953     );
3954
3955 }
3956
3957 # old code, only used to shutdown old VM after update
3958 sub __read_avail {
3959     my ($fh, $timeout) = @_;
3960
3961     my $sel = new IO::Select;
3962     $sel->add($fh);
3963
3964     my $res = '';
3965     my $buf;
3966
3967     my @ready;
3968     while (scalar (@ready = $sel->can_read($timeout))) {
3969         my $count;
3970         if ($count = $fh->sysread($buf, 8192)) {
3971             if ($buf =~ /^(.*)\(qemu\) $/s) {
3972                 $res .= $1;
3973                 last;
3974             } else {
3975                 $res .= $buf;
3976             }
3977         } else {
3978             if (!defined($count)) {
3979                 die "$!\n";
3980             }
3981             last;
3982         }
3983     }
3984
3985     die "monitor read timeout\n" if !scalar(@ready);
3986
3987     return $res;
3988 }
3989
3990 # old code, only used to shutdown old VM after update
3991 sub vm_monitor_command {
3992     my ($vmid, $cmdstr, $nocheck) = @_;
3993
3994     my $res;
3995
3996     eval {
3997         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3998
3999         my $sname = "${var_run_tmpdir}/$vmid.mon";
4000
4001         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
4002             die "unable to connect to VM $vmid socket - $!\n";
4003
4004         my $timeout = 3;
4005
4006         # hack: migrate sometime blocks the monitor (when migrate_downtime
4007         # is set)
4008         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4009             $timeout = 60*60; # 1 hour
4010         }
4011
4012         # read banner;
4013         my $data = __read_avail($sock, $timeout);
4014
4015         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
4016             die "got unexpected qemu monitor banner\n";
4017         }
4018
4019         my $sel = new IO::Select;
4020         $sel->add($sock);
4021
4022         if (!scalar(my @ready = $sel->can_write($timeout))) {
4023             die "monitor write error - timeout";
4024         }
4025
4026         my $fullcmd = "$cmdstr\r";
4027
4028         # syslog('info', "VM $vmid monitor command: $cmdstr");
4029
4030         my $b;
4031         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
4032             die "monitor write error - $!";
4033         }
4034
4035         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4036
4037         $timeout = 20;
4038
4039         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4040             $timeout = 60*60; # 1 hour
4041         } elsif ($cmdstr =~ m/^(eject|change)/) {
4042             $timeout = 60; # note: cdrom mount command is slow
4043         }
4044         if ($res = __read_avail($sock, $timeout)) {
4045
4046             my @lines = split("\r?\n", $res);
4047
4048             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4049
4050             $res = join("\n", @lines);
4051             $res .= "\n";
4052         }
4053     };
4054
4055     my $err = $@;
4056
4057     if ($err) {
4058         syslog("err", "VM $vmid monitor command failed - $err");
4059         die $err;
4060     }
4061
4062     return $res;
4063 }
4064
4065 sub qemu_block_resize {
4066     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4067
4068     my $running = check_running($vmid);
4069
4070     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4071
4072     return if !$running;
4073
4074     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4075
4076 }
4077
4078 sub qemu_volume_snapshot {
4079     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4080
4081     my $running = check_running($vmid);
4082
4083     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4084         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4085     } else {
4086         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4087     }
4088 }
4089
4090 sub qemu_volume_snapshot_delete {
4091     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4092
4093     my $running = check_running($vmid);
4094
4095     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4096         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4097     } else {
4098         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4099     }
4100 }
4101
4102 sub set_migration_caps {
4103     my ($vmid) = @_;
4104
4105     my $cap_ref = [];
4106
4107     my $enabled_cap = {
4108         "auto-converge" => 1,
4109         "xbzrle" => 1,
4110         "x-rdma-pin-all" => 0,
4111         "zero-blocks" => 0,
4112         "compress" => 0
4113     };
4114
4115     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4116
4117     for my $supported_capability (@$supported_capabilities) {
4118         push @$cap_ref, {
4119             capability => $supported_capability->{capability},
4120             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4121         };
4122     }
4123
4124     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4125 }
4126
4127 my $fast_plug_option = {
4128     'lock' => 1,
4129     'name' => 1,
4130     'onboot' => 1,
4131     'shares' => 1,
4132     'startup' => 1,
4133     'description' => 1,
4134     'protection' => 1,
4135     'vmstatestorage' => 1,
4136 };
4137
4138 # hotplug changes in [PENDING]
4139 # $selection hash can be used to only apply specified options, for
4140 # example: { cores => 1 } (only apply changed 'cores')
4141 # $errors ref is used to return error messages
4142 sub vmconfig_hotplug_pending {
4143     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4144
4145     my $defaults = load_defaults();
4146
4147     # commit values which do not have any impact on running VM first
4148     # Note: those option cannot raise errors, we we do not care about
4149     # $selection and always apply them.
4150
4151     my $add_error = sub {
4152         my ($opt, $msg) = @_;
4153         $errors->{$opt} = "hotplug problem - $msg";
4154     };
4155
4156     my $changes = 0;
4157     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4158         if ($fast_plug_option->{$opt}) {
4159             $conf->{$opt} = $conf->{pending}->{$opt};
4160             delete $conf->{pending}->{$opt};
4161             $changes = 1;
4162         }
4163     }
4164
4165     if ($changes) {
4166         PVE::QemuConfig->write_config($vmid, $conf);
4167         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4168     }
4169
4170     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4171
4172     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4173     while (my ($opt, $force) = each %$pending_delete_hash) {
4174         next if $selection && !$selection->{$opt};
4175         eval {
4176             if ($opt eq 'hotplug') {
4177                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4178             } elsif ($opt eq 'tablet') {
4179                 die "skip\n" if !$hotplug_features->{usb};
4180                 if ($defaults->{tablet}) {
4181                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4182                 } else {
4183                     vm_deviceunplug($vmid, $conf, $opt);
4184                 }
4185             } elsif ($opt =~ m/^usb\d+/) {
4186                 die "skip\n";
4187                 # since we cannot reliably hot unplug usb devices
4188                 # we are disabling it
4189                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4190                 vm_deviceunplug($vmid, $conf, $opt);
4191             } elsif ($opt eq 'vcpus') {
4192                 die "skip\n" if !$hotplug_features->{cpu};
4193                 qemu_cpu_hotplug($vmid, $conf, undef);
4194             } elsif ($opt eq 'balloon') {
4195                 # enable balloon device is not hotpluggable
4196                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4197             } elsif ($fast_plug_option->{$opt}) {
4198                 # do nothing
4199             } elsif ($opt =~ m/^net(\d+)$/) {
4200                 die "skip\n" if !$hotplug_features->{network};
4201                 vm_deviceunplug($vmid, $conf, $opt);
4202             } elsif (is_valid_drivename($opt)) {
4203                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4204                 vm_deviceunplug($vmid, $conf, $opt);
4205                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4206             } elsif ($opt =~ m/^memory$/) {
4207                 die "skip\n" if !$hotplug_features->{memory};
4208                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4209             } elsif ($opt eq 'cpuunits') {
4210                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4211             } elsif ($opt eq 'cpulimit') {
4212                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4213             } else {
4214                 die "skip\n";
4215             }
4216         };
4217         if (my $err = $@) {
4218             &$add_error($opt, $err) if $err ne "skip\n";
4219         } else {
4220             # save new config if hotplug was successful
4221             delete $conf->{$opt};
4222             vmconfig_undelete_pending_option($conf, $opt);
4223             PVE::QemuConfig->write_config($vmid, $conf);
4224             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4225         }
4226     }
4227
4228     foreach my $opt (keys %{$conf->{pending}}) {
4229         next if $selection && !$selection->{$opt};
4230         my $value = $conf->{pending}->{$opt};
4231         eval {
4232             if ($opt eq 'hotplug') {
4233                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4234             } elsif ($opt eq 'tablet') {
4235                 die "skip\n" if !$hotplug_features->{usb};
4236                 if ($value == 1) {
4237                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4238                 } elsif ($value == 0) {
4239                     vm_deviceunplug($vmid, $conf, $opt);
4240                 }
4241             } elsif ($opt =~ m/^usb\d+$/) {
4242                 die "skip\n";
4243                 # since we cannot reliably hot unplug usb devices
4244                 # we are disabling it
4245                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4246                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4247                 die "skip\n" if !$d;
4248                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4249             } elsif ($opt eq 'vcpus') {
4250                 die "skip\n" if !$hotplug_features->{cpu};
4251                 qemu_cpu_hotplug($vmid, $conf, $value);
4252             } elsif ($opt eq 'balloon') {
4253                 # enable/disable balloning device is not hotpluggable
4254                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4255                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4256                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4257
4258                 # allow manual ballooning if shares is set to zero
4259                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4260                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4261                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4262                 }
4263             } elsif ($opt =~ m/^net(\d+)$/) {
4264                 # some changes can be done without hotplug
4265                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4266                                     $vmid, $opt, $value);
4267             } elsif (is_valid_drivename($opt)) {
4268                 # some changes can be done without hotplug
4269                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4270                                      $vmid, $opt, $value, 1);
4271             } elsif ($opt =~ m/^memory$/) { #dimms
4272                 die "skip\n" if !$hotplug_features->{memory};
4273                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4274             } elsif ($opt eq 'cpuunits') {
4275                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4276             } elsif ($opt eq 'cpulimit') {
4277                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4278                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4279             } else {
4280                 die "skip\n";  # skip non-hot-pluggable options
4281             }
4282         };
4283         if (my $err = $@) {
4284             &$add_error($opt, $err) if $err ne "skip\n";
4285         } else {
4286             # save new config if hotplug was successful
4287             $conf->{$opt} = $value;
4288             delete $conf->{pending}->{$opt};
4289             PVE::QemuConfig->write_config($vmid, $conf);
4290             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4291         }
4292     }
4293 }
4294
4295 sub try_deallocate_drive {
4296     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4297
4298     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4299         my $volid = $drive->{file};
4300         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4301             my $sid = PVE::Storage::parse_volume_id($volid);
4302             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4303
4304             # check if the disk is really unused
4305             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4306                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4307             PVE::Storage::vdisk_free($storecfg, $volid);
4308             return 1;
4309         } else {
4310             # If vm is not owner of this disk remove from config
4311             return 1;
4312         }
4313     }
4314
4315     return undef;
4316 }
4317
4318 sub vmconfig_delete_or_detach_drive {
4319     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4320
4321     my $drive = parse_drive($opt, $conf->{$opt});
4322
4323     my $rpcenv = PVE::RPCEnvironment::get();
4324     my $authuser = $rpcenv->get_user();
4325
4326     if ($force) {
4327         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4328         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4329     } else {
4330         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4331     }
4332 }
4333
4334 sub vmconfig_apply_pending {
4335     my ($vmid, $conf, $storecfg) = @_;
4336
4337     # cold plug
4338
4339     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4340     while (my ($opt, $force) = each %$pending_delete_hash) {
4341         die "internal error" if $opt =~ m/^unused/;
4342         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4343         if (!defined($conf->{$opt})) {
4344             vmconfig_undelete_pending_option($conf, $opt);
4345             PVE::QemuConfig->write_config($vmid, $conf);
4346         } elsif (is_valid_drivename($opt)) {
4347             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4348             vmconfig_undelete_pending_option($conf, $opt);
4349             delete $conf->{$opt};
4350             PVE::QemuConfig->write_config($vmid, $conf);
4351         } else {
4352             vmconfig_undelete_pending_option($conf, $opt);
4353             delete $conf->{$opt};
4354             PVE::QemuConfig->write_config($vmid, $conf);
4355         }
4356     }
4357
4358     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4359
4360     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4361         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4362
4363         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4364             # skip if nothing changed
4365         } elsif (is_valid_drivename($opt)) {
4366             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4367                 if defined($conf->{$opt});
4368             $conf->{$opt} = $conf->{pending}->{$opt};
4369         } else {
4370             $conf->{$opt} = $conf->{pending}->{$opt};
4371         }
4372
4373         delete $conf->{pending}->{$opt};
4374         PVE::QemuConfig->write_config($vmid, $conf);
4375     }
4376 }
4377
4378 my $safe_num_ne = sub {
4379     my ($a, $b) = @_;
4380
4381     return 0 if !defined($a) && !defined($b);
4382     return 1 if !defined($a);
4383     return 1 if !defined($b);
4384
4385     return $a != $b;
4386 };
4387
4388 my $safe_string_ne = sub {
4389     my ($a, $b) = @_;
4390
4391     return 0 if !defined($a) && !defined($b);
4392     return 1 if !defined($a);
4393     return 1 if !defined($b);
4394
4395     return $a ne $b;
4396 };
4397
4398 sub vmconfig_update_net {
4399     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4400
4401     my $newnet = parse_net($value);
4402
4403     if ($conf->{$opt}) {
4404         my $oldnet = parse_net($conf->{$opt});
4405
4406         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4407             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4408             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4409             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4410
4411             # for non online change, we try to hot-unplug
4412             die "skip\n" if !$hotplug;
4413             vm_deviceunplug($vmid, $conf, $opt);
4414         } else {
4415
4416             die "internal error" if $opt !~ m/net(\d+)/;
4417             my $iface = "tap${vmid}i$1";
4418
4419             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4420                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4421                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4422                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4423                 PVE::Network::tap_unplug($iface);
4424                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4425             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4426                 # Rate can be applied on its own but any change above needs to
4427                 # include the rate in tap_plug since OVS resets everything.
4428                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4429             }
4430
4431             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4432                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4433             }
4434
4435             return 1;
4436         }
4437     }
4438
4439     if ($hotplug) {
4440         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4441     } else {
4442         die "skip\n";
4443     }
4444 }
4445
4446 sub vmconfig_update_disk {
4447     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4448
4449     # fixme: do we need force?
4450
4451     my $drive = parse_drive($opt, $value);
4452
4453     if ($conf->{$opt}) {
4454
4455         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4456
4457             my $media = $drive->{media} || 'disk';
4458             my $oldmedia = $old_drive->{media} || 'disk';
4459             die "unable to change media type\n" if $media ne $oldmedia;
4460
4461             if (!drive_is_cdrom($old_drive)) {
4462
4463                 if ($drive->{file} ne $old_drive->{file}) {
4464
4465                     die "skip\n" if !$hotplug;
4466
4467                     # unplug and register as unused
4468                     vm_deviceunplug($vmid, $conf, $opt);
4469                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4470
4471                 } else {
4472                     # update existing disk
4473
4474                     # skip non hotpluggable value
4475                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4476                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4477                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4478                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4479                         die "skip\n";
4480                     }
4481
4482                     # apply throttle
4483                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4484                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4485                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4486                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4487                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4488                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4489                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4490                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4491                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4492                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4493                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4494                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4495                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4496                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4497                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4498                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4499                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4500                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4501
4502                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4503                                                    ($drive->{mbps} || 0)*1024*1024,
4504                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4505                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4506                                                    $drive->{iops} || 0,
4507                                                    $drive->{iops_rd} || 0,
4508                                                    $drive->{iops_wr} || 0,
4509                                                    ($drive->{mbps_max} || 0)*1024*1024,
4510                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4511                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4512                                                    $drive->{iops_max} || 0,
4513                                                    $drive->{iops_rd_max} || 0,
4514                                                    $drive->{iops_wr_max} || 0,
4515                                                    $drive->{bps_max_length} || 1,
4516                                                    $drive->{bps_rd_max_length} || 1,
4517                                                    $drive->{bps_wr_max_length} || 1,
4518                                                    $drive->{iops_max_length} || 1,
4519                                                    $drive->{iops_rd_max_length} || 1,
4520                                                    $drive->{iops_wr_max_length} || 1);
4521
4522                     }
4523
4524                     return 1;
4525                 }
4526
4527             } else { # cdrom
4528
4529                 if ($drive->{file} eq 'none') {
4530                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4531                 } else {
4532                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4533                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4534                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4535                 }
4536
4537                 return 1;
4538             }
4539         }
4540     }
4541
4542     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4543     # hotplug new disks
4544     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4545     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4546 }
4547
4548 sub vm_start {
4549     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4550         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4551
4552     PVE::QemuConfig->lock_config($vmid, sub {
4553         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4554
4555         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4556
4557         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4558
4559         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4560
4561         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4562             vmconfig_apply_pending($vmid, $conf, $storecfg);
4563             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4564         }
4565
4566         my $defaults = load_defaults();
4567
4568         # set environment variable useful inside network script
4569         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4570
4571         my $local_volumes = {};
4572
4573         if ($targetstorage) {
4574             foreach_drive($conf, sub {
4575                 my ($ds, $drive) = @_;
4576
4577                 return if drive_is_cdrom($drive);
4578
4579                 my $volid = $drive->{file};
4580
4581                 return if !$volid;
4582
4583                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4584
4585                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4586                 return if $scfg->{shared};
4587                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4588             });
4589
4590             my $format = undef;
4591
4592             foreach my $opt (sort keys %$local_volumes) {
4593
4594                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4595                 my $drive = parse_drive($opt, $conf->{$opt});
4596
4597                 #if remote storage is specified, use default format
4598                 if ($targetstorage && $targetstorage ne "1") {
4599                     $storeid = $targetstorage;
4600                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4601                     $format = $defFormat;
4602                 } else {
4603                     #else we use same format than original
4604                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4605                     $format = qemu_img_format($scfg, $volid);
4606                 }
4607
4608                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4609                 my $newdrive = $drive;
4610                 $newdrive->{format} = $format;
4611                 $newdrive->{file} = $newvolid;
4612                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4613                 $local_volumes->{$opt} = $drivestr;
4614                 #pass drive to conf for command line
4615                 $conf->{$opt} = $drivestr;
4616             }
4617         }
4618
4619         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4620
4621         my $migrate_port = 0;
4622         my $migrate_uri;
4623         if ($statefile) {
4624             if ($statefile eq 'tcp') {
4625                 my $localip = "localhost";
4626                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4627                 my $nodename = PVE::INotify::nodename();
4628
4629                 if (!defined($migration_type)) {
4630                     if (defined($datacenterconf->{migration}->{type})) {
4631                         $migration_type = $datacenterconf->{migration}->{type};
4632                     } else {
4633                         $migration_type = 'secure';
4634                     }
4635                 }
4636
4637                 if ($migration_type eq 'insecure') {
4638                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4639                     if ($migrate_network_addr) {
4640                         $localip = $migrate_network_addr;
4641                     } else {
4642                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4643                     }
4644
4645                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4646                 }
4647
4648                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4649                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4650                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4651                 push @$cmd, '-incoming', $migrate_uri;
4652                 push @$cmd, '-S';
4653
4654             } elsif ($statefile eq 'unix') {
4655                 # should be default for secure migrations as a ssh TCP forward
4656                 # tunnel is not deterministic reliable ready and fails regurarly
4657                 # to set up in time, so use UNIX socket forwards
4658                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4659                 unlink $socket_addr;
4660
4661                 $migrate_uri = "unix:$socket_addr";
4662
4663                 push @$cmd, '-incoming', $migrate_uri;
4664                 push @$cmd, '-S';
4665
4666             } else {
4667                 push @$cmd, '-loadstate', $statefile;
4668             }
4669         } elsif ($paused) {
4670             push @$cmd, '-S';
4671         }
4672
4673         # host pci devices
4674         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4675           my $d = parse_hostpci($conf->{"hostpci$i"});
4676           next if !$d;
4677           my $pcidevices = $d->{pciid};
4678           foreach my $pcidevice (@$pcidevices) {
4679                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4680
4681                 my $info = pci_device_info("0000:$pciid");
4682                 die "IOMMU not present\n" if !check_iommu_support();
4683                 die "no pci device info for device '$pciid'\n" if !$info;
4684                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4685                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4686           }
4687         }
4688
4689         PVE::Storage::activate_volumes($storecfg, $vollist);
4690
4691         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4692             my $cmd = [];
4693             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4694             eval  { run_command($cmd); };
4695         }
4696
4697         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4698                                                   : $defaults->{cpuunits};
4699
4700         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4701         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4702
4703         my %properties = (
4704             Slice => 'qemu.slice',
4705             KillMode => 'none',
4706             CPUShares => $cpuunits
4707         );
4708
4709         if (my $cpulimit = $conf->{cpulimit}) {
4710             $properties{CPUQuota} = int($cpulimit * 100);
4711         }
4712         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4713
4714         if ($conf->{hugepages}) {
4715
4716             my $code = sub {
4717                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4718                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4719
4720                 PVE::QemuServer::Memory::hugepages_mount();
4721                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4722
4723                 eval  {
4724                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4725                     run_command($cmd, %run_params);
4726                 };
4727
4728                 if (my $err = $@) {
4729                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4730                     die $err;
4731                 }
4732
4733                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4734             };
4735             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4736
4737         } else {
4738             eval  {
4739                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4740                 run_command($cmd, %run_params);
4741             };
4742         }
4743
4744         if (my $err = $@) {
4745             # deactivate volumes if start fails
4746             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4747             die "start failed: $err";
4748         }
4749
4750         print "migration listens on $migrate_uri\n" if $migrate_uri;
4751
4752         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4753             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4754             warn $@ if $@;
4755         }
4756
4757         #start nbd server for storage migration
4758         if ($targetstorage) {
4759             my $nodename = PVE::INotify::nodename();
4760             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4761             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4762             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4763             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4764
4765             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4766
4767             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4768
4769             foreach my $opt (sort keys %$local_volumes) {
4770                 my $volid = $local_volumes->{$opt};
4771                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4772                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4773                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4774             }
4775         }
4776
4777         if ($migratedfrom) {
4778             eval {
4779                 set_migration_caps($vmid);
4780             };
4781             warn $@ if $@;
4782
4783             if ($spice_port) {
4784                 print "spice listens on port $spice_port\n";
4785                 if ($spice_ticket) {
4786                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4787                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4788                 }
4789             }
4790
4791         } else {
4792             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4793                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4794                     if $conf->{balloon};
4795             }
4796
4797             foreach my $opt (keys %$conf) {
4798                 next if $opt !~  m/^net\d+$/;
4799                 my $nicconf = parse_net($conf->{$opt});
4800                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4801             }
4802         }
4803
4804         vm_mon_cmd_nocheck($vmid, 'qom-set',
4805                     path => "machine/peripheral/balloon0",
4806                     property => "guest-stats-polling-interval",
4807                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4808
4809     });
4810 }
4811
4812 sub vm_mon_cmd {
4813     my ($vmid, $execute, %params) = @_;
4814
4815     my $cmd = { execute => $execute, arguments => \%params };
4816     vm_qmp_command($vmid, $cmd);
4817 }
4818
4819 sub vm_mon_cmd_nocheck {
4820     my ($vmid, $execute, %params) = @_;
4821
4822     my $cmd = { execute => $execute, arguments => \%params };
4823     vm_qmp_command($vmid, $cmd, 1);
4824 }
4825
4826 sub vm_qmp_command {
4827     my ($vmid, $cmd, $nocheck) = @_;
4828
4829     my $res;
4830
4831     my $timeout;
4832     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4833         $timeout = $cmd->{arguments}->{timeout};
4834         delete $cmd->{arguments}->{timeout};
4835     }
4836
4837     eval {
4838         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4839         my $sname = qmp_socket($vmid);
4840         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4841             my $qmpclient = PVE::QMPClient->new();
4842
4843             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4844         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4845             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4846                 if scalar(%{$cmd->{arguments}});
4847             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4848         } else {
4849             die "unable to open monitor socket\n";
4850         }
4851     };
4852     if (my $err = $@) {
4853         syslog("err", "VM $vmid qmp command failed - $err");
4854         die $err;
4855     }
4856
4857     return $res;
4858 }
4859
4860 sub vm_human_monitor_command {
4861     my ($vmid, $cmdline) = @_;
4862
4863     my $res;
4864
4865     my $cmd = {
4866         execute => 'human-monitor-command',
4867         arguments => { 'command-line' => $cmdline},
4868     };
4869
4870     return vm_qmp_command($vmid, $cmd);
4871 }
4872
4873 sub vm_commandline {
4874     my ($storecfg, $vmid) = @_;
4875
4876     my $conf = PVE::QemuConfig->load_config($vmid);
4877
4878     my $defaults = load_defaults();
4879
4880     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4881
4882     return PVE::Tools::cmd2string($cmd);
4883 }
4884
4885 sub vm_reset {
4886     my ($vmid, $skiplock) = @_;
4887
4888     PVE::QemuConfig->lock_config($vmid, sub {
4889
4890         my $conf = PVE::QemuConfig->load_config($vmid);
4891
4892         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4893
4894         vm_mon_cmd($vmid, "system_reset");
4895     });
4896 }
4897
4898 sub get_vm_volumes {
4899     my ($conf) = @_;
4900
4901     my $vollist = [];
4902     foreach_volid($conf, sub {
4903         my ($volid, $attr) = @_;
4904
4905         return if $volid =~ m|^/|;
4906
4907         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4908         return if !$sid;
4909
4910         push @$vollist, $volid;
4911     });
4912
4913     return $vollist;
4914 }
4915
4916 sub vm_stop_cleanup {
4917     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4918
4919     eval {
4920
4921         if (!$keepActive) {
4922             my $vollist = get_vm_volumes($conf);
4923             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4924         }
4925
4926         foreach my $ext (qw(mon qmp pid vnc qga)) {
4927             unlink "/var/run/qemu-server/${vmid}.$ext";
4928         }
4929
4930         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4931     };
4932     warn $@ if $@; # avoid errors - just warn
4933 }
4934
4935 # Note: use $nockeck to skip tests if VM configuration file exists.
4936 # We need that when migration VMs to other nodes (files already moved)
4937 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4938 sub vm_stop {
4939     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4940
4941     $force = 1 if !defined($force) && !$shutdown;
4942
4943     if ($migratedfrom){
4944         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4945         kill 15, $pid if $pid;
4946         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4947         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4948         return;
4949     }
4950
4951     PVE::QemuConfig->lock_config($vmid, sub {
4952
4953         my $pid = check_running($vmid, $nocheck);
4954         return if !$pid;
4955
4956         my $conf;
4957         if (!$nocheck) {
4958             $conf = PVE::QemuConfig->load_config($vmid);
4959             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4960             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4961                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4962                 $timeout = $opts->{down} if $opts->{down};
4963             }
4964         }
4965
4966         $timeout = 60 if !defined($timeout);
4967
4968         eval {
4969             if ($shutdown) {
4970                 if (defined($conf) && $conf->{agent}) {
4971                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4972                 } else {
4973                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4974                 }
4975             } else {
4976                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4977             }
4978         };
4979         my $err = $@;
4980
4981         if (!$err) {
4982             my $count = 0;
4983             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4984                 $count++;
4985                 sleep 1;
4986             }
4987
4988             if ($count >= $timeout) {
4989                 if ($force) {
4990                     warn "VM still running - terminating now with SIGTERM\n";
4991                     kill 15, $pid;
4992                 } else {
4993                     die "VM quit/powerdown failed - got timeout\n";
4994                 }
4995             } else {
4996                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4997                 return;
4998             }
4999         } else {
5000             if ($force) {
5001                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5002                 kill 15, $pid;
5003             } else {
5004                 die "VM quit/powerdown failed\n";
5005             }
5006         }
5007
5008         # wait again
5009         $timeout = 10;
5010
5011         my $count = 0;
5012         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5013             $count++;
5014             sleep 1;
5015         }
5016
5017         if ($count >= $timeout) {
5018             warn "VM still running - terminating now with SIGKILL\n";
5019             kill 9, $pid;
5020             sleep 1;
5021         }
5022
5023         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5024    });
5025 }
5026
5027 sub vm_suspend {
5028     my ($vmid, $skiplock) = @_;
5029
5030     PVE::QemuConfig->lock_config($vmid, sub {
5031
5032         my $conf = PVE::QemuConfig->load_config($vmid);
5033
5034         PVE::QemuConfig->check_lock($conf)
5035             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5036
5037         vm_mon_cmd($vmid, "stop");
5038     });
5039 }
5040
5041 sub vm_resume {
5042     my ($vmid, $skiplock, $nocheck) = @_;
5043
5044     PVE::QemuConfig->lock_config($vmid, sub {
5045
5046         if (!$nocheck) {
5047
5048             my $conf = PVE::QemuConfig->load_config($vmid);
5049
5050             PVE::QemuConfig->check_lock($conf)
5051                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5052
5053             vm_mon_cmd($vmid, "cont");
5054
5055         } else {
5056             vm_mon_cmd_nocheck($vmid, "cont");
5057         }
5058     });
5059 }
5060
5061 sub vm_sendkey {
5062     my ($vmid, $skiplock, $key) = @_;
5063
5064     PVE::QemuConfig->lock_config($vmid, sub {
5065
5066         my $conf = PVE::QemuConfig->load_config($vmid);
5067
5068         # there is no qmp command, so we use the human monitor command
5069         vm_human_monitor_command($vmid, "sendkey $key");
5070     });
5071 }
5072
5073 sub vm_destroy {
5074     my ($storecfg, $vmid, $skiplock) = @_;
5075
5076     PVE::QemuConfig->lock_config($vmid, sub {
5077
5078         my $conf = PVE::QemuConfig->load_config($vmid);
5079
5080         if (!check_running($vmid)) {
5081             destroy_vm($storecfg, $vmid, undef, $skiplock);
5082         } else {
5083             die "VM $vmid is running - destroy failed\n";
5084         }
5085     });
5086 }
5087
5088 # pci helpers
5089
5090 sub file_write {
5091     my ($filename, $buf) = @_;
5092
5093     my $fh = IO::File->new($filename, "w");
5094     return undef if !$fh;
5095
5096     my $res = print $fh $buf;
5097
5098     $fh->close();
5099
5100     return $res;
5101 }
5102
5103 sub pci_device_info {
5104     my ($name) = @_;
5105
5106     my $res;
5107
5108     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5109     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5110
5111     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5112     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5113
5114     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5115     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5116
5117     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5118     return undef if !defined($product) || $product !~ s/^0x//;
5119
5120     $res = {
5121         name => $name,
5122         vendor => $vendor,
5123         product => $product,
5124         domain => $domain,
5125         bus => $bus,
5126         slot => $slot,
5127         func => $func,
5128         irq => $irq,
5129         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5130     };
5131
5132     return $res;
5133 }
5134
5135 sub pci_dev_reset {
5136     my ($dev) = @_;
5137
5138     my $name = $dev->{name};
5139
5140     my $fn = "$pcisysfs/devices/$name/reset";
5141
5142     return file_write($fn, "1");
5143 }
5144
5145 sub pci_dev_bind_to_vfio {
5146     my ($dev) = @_;
5147
5148     my $name = $dev->{name};
5149
5150     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5151
5152     if (!-d $vfio_basedir) {
5153         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5154     }
5155     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5156
5157     my $testdir = "$vfio_basedir/$name";
5158     return 1 if -d $testdir;
5159
5160     my $data = "$dev->{vendor} $dev->{product}";
5161     return undef if !file_write("$vfio_basedir/new_id", $data);
5162
5163     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5164     if (!file_write($fn, $name)) {
5165         return undef if -f $fn;
5166     }
5167
5168     $fn = "$vfio_basedir/bind";
5169     if (! -d $testdir) {
5170         return undef if !file_write($fn, $name);
5171     }
5172
5173     return -d $testdir;
5174 }
5175
5176 sub pci_dev_group_bind_to_vfio {
5177     my ($pciid) = @_;
5178
5179     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5180
5181     if (!-d $vfio_basedir) {
5182         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5183     }
5184     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5185
5186     # get IOMMU group devices
5187     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5188       my @devs = grep /^0000:/, readdir($D);
5189     closedir($D);
5190
5191     foreach my $pciid (@devs) {
5192         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5193
5194         # pci bridges, switches or root ports are not supported
5195         # they have a pci_bus subdirectory so skip them
5196         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5197
5198         my $info = pci_device_info($1);
5199         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5200     }
5201
5202     return 1;
5203 }
5204
5205 # vzdump restore implementaion
5206
5207 sub tar_archive_read_firstfile {
5208     my $archive = shift;
5209
5210     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5211
5212     # try to detect archive type first
5213     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5214         die "unable to open file '$archive'\n";
5215     my $firstfile = <$fh>;
5216     kill 15, $pid;
5217     close $fh;
5218
5219     die "ERROR: archive contaions no data\n" if !$firstfile;
5220     chomp $firstfile;
5221
5222     return $firstfile;
5223 }
5224
5225 sub tar_restore_cleanup {
5226     my ($storecfg, $statfile) = @_;
5227
5228     print STDERR "starting cleanup\n";
5229
5230     if (my $fd = IO::File->new($statfile, "r")) {
5231         while (defined(my $line = <$fd>)) {
5232             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5233                 my $volid = $2;
5234                 eval {
5235                     if ($volid =~ m|^/|) {
5236                         unlink $volid || die 'unlink failed\n';
5237                     } else {
5238                         PVE::Storage::vdisk_free($storecfg, $volid);
5239                     }
5240                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5241                 };
5242                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5243             } else {
5244                 print STDERR "unable to parse line in statfile - $line";
5245             }
5246         }
5247         $fd->close();
5248     }
5249 }
5250
5251 sub restore_archive {
5252     my ($archive, $vmid, $user, $opts) = @_;
5253
5254     my $format = $opts->{format};
5255     my $comp;
5256
5257     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5258         $format = 'tar' if !$format;
5259         $comp = 'gzip';
5260     } elsif ($archive =~ m/\.tar$/) {
5261         $format = 'tar' if !$format;
5262     } elsif ($archive =~ m/.tar.lzo$/) {
5263         $format = 'tar' if !$format;
5264         $comp = 'lzop';
5265     } elsif ($archive =~ m/\.vma$/) {
5266         $format = 'vma' if !$format;
5267     } elsif ($archive =~ m/\.vma\.gz$/) {
5268         $format = 'vma' if !$format;
5269         $comp = 'gzip';
5270     } elsif ($archive =~ m/\.vma\.lzo$/) {
5271         $format = 'vma' if !$format;
5272         $comp = 'lzop';
5273     } else {
5274         $format = 'vma' if !$format; # default
5275     }
5276
5277     # try to detect archive format
5278     if ($format eq 'tar') {
5279         return restore_tar_archive($archive, $vmid, $user, $opts);
5280     } else {
5281         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5282     }
5283 }
5284
5285 sub restore_update_config_line {
5286     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5287
5288     return if $line =~ m/^\#qmdump\#/;
5289     return if $line =~ m/^\#vzdump\#/;
5290     return if $line =~ m/^lock:/;
5291     return if $line =~ m/^unused\d+:/;
5292     return if $line =~ m/^parent:/;
5293     return if $line =~ m/^template:/; # restored VM is never a template
5294
5295     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5296     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5297         # try to convert old 1.X settings
5298         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5299         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5300             my ($model, $macaddr) = split(/\=/, $devconfig);
5301             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5302             my $net = {
5303                 model => $model,
5304                 bridge => "vmbr$ind",
5305                 macaddr => $macaddr,
5306             };
5307             my $netstr = print_net($net);
5308
5309             print $outfd "net$cookie->{netcount}: $netstr\n";
5310             $cookie->{netcount}++;
5311         }
5312     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5313         my ($id, $netstr) = ($1, $2);
5314         my $net = parse_net($netstr);
5315         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5316         $netstr = print_net($net);
5317         print $outfd "$id: $netstr\n";
5318     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5319         my $virtdev = $1;
5320         my $value = $3;
5321         my $di = parse_drive($virtdev, $value);
5322         if (defined($di->{backup}) && !$di->{backup}) {
5323             print $outfd "#$line";
5324         } elsif ($map->{$virtdev}) {
5325             delete $di->{format}; # format can change on restore
5326             $di->{file} = $map->{$virtdev};
5327             $value = print_drive($vmid, $di);
5328             print $outfd "$virtdev: $value\n";
5329         } else {
5330             print $outfd $line;
5331         }
5332     } else {
5333         print $outfd $line;
5334     }
5335 }
5336
5337 sub scan_volids {
5338     my ($cfg, $vmid) = @_;
5339
5340     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5341
5342     my $volid_hash = {};
5343     foreach my $storeid (keys %$info) {
5344         foreach my $item (@{$info->{$storeid}}) {
5345             next if !($item->{volid} && $item->{size});
5346             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5347             $volid_hash->{$item->{volid}} = $item;
5348         }
5349     }
5350
5351     return $volid_hash;
5352 }
5353
5354 sub is_volume_in_use {
5355     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5356
5357     my $path = PVE::Storage::path($storecfg, $volid);
5358
5359     my $scan_config = sub {
5360         my ($cref, $snapname) = @_;
5361
5362         foreach my $key (keys %$cref) {
5363             my $value = $cref->{$key};
5364             if (is_valid_drivename($key)) {
5365                 next if $skip_drive && $key eq $skip_drive;
5366                 my $drive = parse_drive($key, $value);
5367                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5368                 return 1 if $volid eq $drive->{file};
5369                 if ($drive->{file} =~ m!^/!) {
5370                     return 1 if $drive->{file} eq $path;
5371                 } else {
5372                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5373                     next if !$storeid;
5374                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5375                     next if !$scfg;
5376                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5377                 }
5378             }
5379         }
5380
5381         return 0;
5382     };
5383
5384     return 1 if &$scan_config($conf);
5385
5386     undef $skip_drive;
5387
5388     foreach my $snapname (keys %{$conf->{snapshots}}) {
5389         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5390     }
5391
5392     return 0;
5393 }
5394
5395 sub update_disksize {
5396     my ($vmid, $conf, $volid_hash) = @_;
5397
5398     my $changes;
5399
5400     # used and unused disks
5401     my $referenced = {};
5402
5403     # Note: it is allowed to define multiple storages with same path (alias), so
5404     # we need to check both 'volid' and real 'path' (two different volid can point
5405     # to the same path).
5406
5407     my $referencedpath = {};
5408
5409     # update size info
5410     foreach my $opt (keys %$conf) {
5411         if (is_valid_drivename($opt)) {
5412             my $drive = parse_drive($opt, $conf->{$opt});
5413             my $volid = $drive->{file};
5414             next if !$volid;
5415
5416             $referenced->{$volid} = 1;
5417             if ($volid_hash->{$volid} &&
5418                 (my $path = $volid_hash->{$volid}->{path})) {
5419                 $referencedpath->{$path} = 1;
5420             }
5421
5422             next if drive_is_cdrom($drive);
5423             next if !$volid_hash->{$volid};
5424
5425             $drive->{size} = $volid_hash->{$volid}->{size};
5426             my $new = print_drive($vmid, $drive);
5427             if ($new ne $conf->{$opt}) {
5428                 $changes = 1;
5429                 $conf->{$opt} = $new;
5430             }
5431         }
5432     }
5433
5434     # remove 'unusedX' entry if volume is used
5435     foreach my $opt (keys %$conf) {
5436         next if $opt !~ m/^unused\d+$/;
5437         my $volid = $conf->{$opt};
5438         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5439         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5440             $changes = 1;
5441             delete $conf->{$opt};
5442         }
5443
5444         $referenced->{$volid} = 1;
5445         $referencedpath->{$path} = 1 if $path;
5446     }
5447
5448     foreach my $volid (sort keys %$volid_hash) {
5449         next if $volid =~ m/vm-$vmid-state-/;
5450         next if $referenced->{$volid};
5451         my $path = $volid_hash->{$volid}->{path};
5452         next if !$path; # just to be sure
5453         next if $referencedpath->{$path};
5454         $changes = 1;
5455         PVE::QemuConfig->add_unused_volume($conf, $volid);
5456         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5457     }
5458
5459     return $changes;
5460 }
5461
5462 sub rescan {
5463     my ($vmid, $nolock) = @_;
5464
5465     my $cfg = PVE::Storage::config();
5466
5467     my $volid_hash = scan_volids($cfg, $vmid);
5468
5469     my $updatefn =  sub {
5470         my ($vmid) = @_;
5471
5472         my $conf = PVE::QemuConfig->load_config($vmid);
5473
5474         PVE::QemuConfig->check_lock($conf);
5475
5476         my $vm_volids = {};
5477         foreach my $volid (keys %$volid_hash) {
5478             my $info = $volid_hash->{$volid};
5479             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5480         }
5481
5482         my $changes = update_disksize($vmid, $conf, $vm_volids);
5483
5484         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5485     };
5486
5487     if (defined($vmid)) {
5488         if ($nolock) {
5489             &$updatefn($vmid);
5490         } else {
5491             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5492         }
5493     } else {
5494         my $vmlist = config_list();
5495         foreach my $vmid (keys %$vmlist) {
5496             if ($nolock) {
5497                 &$updatefn($vmid);
5498             } else {
5499                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5500             }
5501         }
5502     }
5503 }
5504
5505 sub restore_vma_archive {
5506     my ($archive, $vmid, $user, $opts, $comp) = @_;
5507
5508     my $input = $archive eq '-' ? "<&STDIN" : undef;
5509     my $readfrom = $archive;
5510
5511     my $uncomp = '';
5512     if ($comp) {
5513         $readfrom = '-';
5514         my $qarchive = PVE::Tools::shellquote($archive);
5515         if ($comp eq 'gzip') {
5516             $uncomp = "zcat $qarchive|";
5517         } elsif ($comp eq 'lzop') {
5518             $uncomp = "lzop -d -c $qarchive|";
5519         } else {
5520             die "unknown compression method '$comp'\n";
5521         }
5522
5523     }
5524
5525     my $tmpdir = "/var/tmp/vzdumptmp$$";
5526     rmtree $tmpdir;
5527
5528     # disable interrupts (always do cleanups)
5529     local $SIG{INT} =
5530         local $SIG{TERM} =
5531         local $SIG{QUIT} =
5532         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5533
5534     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5535     POSIX::mkfifo($mapfifo, 0600);
5536     my $fifofh;
5537
5538     my $openfifo = sub {
5539         open($fifofh, '>', $mapfifo) || die $!;
5540     };
5541
5542     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5543
5544     my $oldtimeout;
5545     my $timeout = 5;
5546
5547     my $devinfo = {};
5548
5549     my $rpcenv = PVE::RPCEnvironment::get();
5550
5551     my $conffile = PVE::QemuConfig->config_file($vmid);
5552     my $tmpfn = "$conffile.$$.tmp";
5553
5554     # Note: $oldconf is undef if VM does not exists
5555     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5556     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5557
5558     my $print_devmap = sub {
5559         my $virtdev_hash = {};
5560
5561         my $cfgfn = "$tmpdir/qemu-server.conf";
5562
5563         # we can read the config - that is already extracted
5564         my $fh = IO::File->new($cfgfn, "r") ||
5565             "unable to read qemu-server.conf - $!\n";
5566
5567         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5568         if (-f $fwcfgfn) {
5569             my $pve_firewall_dir = '/etc/pve/firewall';
5570             mkdir $pve_firewall_dir; # make sure the dir exists
5571             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5572         }
5573
5574         while (defined(my $line = <$fh>)) {
5575             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5576                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5577                 die "archive does not contain data for drive '$virtdev'\n"
5578                     if !$devinfo->{$devname};
5579                 if (defined($opts->{storage})) {
5580                     $storeid = $opts->{storage} || 'local';
5581                 } elsif (!$storeid) {
5582                     $storeid = 'local';
5583                 }
5584                 $format = 'raw' if !$format;
5585                 $devinfo->{$devname}->{devname} = $devname;
5586                 $devinfo->{$devname}->{virtdev} = $virtdev;
5587                 $devinfo->{$devname}->{format} = $format;
5588                 $devinfo->{$devname}->{storeid} = $storeid;
5589
5590                 # check permission on storage
5591                 my $pool = $opts->{pool}; # todo: do we need that?
5592                 if ($user ne 'root@pam') {
5593                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5594                 }
5595
5596                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5597             }
5598         }
5599
5600         foreach my $devname (keys %$devinfo) {
5601             die "found no device mapping information for device '$devname'\n"
5602                 if !$devinfo->{$devname}->{virtdev};
5603         }
5604
5605         my $cfg = PVE::Storage::config();
5606
5607         # create empty/temp config
5608         if ($oldconf) {
5609             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5610             foreach_drive($oldconf, sub {
5611                 my ($ds, $drive) = @_;
5612
5613                 return if drive_is_cdrom($drive);
5614
5615                 my $volid = $drive->{file};
5616
5617                 return if !$volid || $volid =~ m|^/|;
5618
5619                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5620                 return if !$path || !$owner || ($owner != $vmid);
5621
5622                 # Note: only delete disk we want to restore
5623                 # other volumes will become unused
5624                 if ($virtdev_hash->{$ds}) {
5625                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5626                     if (my $err = $@) {
5627                         warn $err;
5628                     }
5629                 }
5630             });
5631
5632             # delete vmstate files
5633             # since after the restore we have no snapshots anymore
5634             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5635                 my $snap = $oldconf->{snapshots}->{$snapname};
5636                 if ($snap->{vmstate}) {
5637                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5638                     if (my $err = $@) {
5639                         warn $err;
5640                     }
5641                 }
5642             }
5643         }
5644
5645         my $map = {};
5646         foreach my $virtdev (sort keys %$virtdev_hash) {
5647             my $d = $virtdev_hash->{$virtdev};
5648             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5649             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5650
5651             # test if requested format is supported
5652             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5653             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5654             $d->{format} = $defFormat if !$supported;
5655
5656             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5657                                                   $d->{format}, undef, $alloc_size);
5658             print STDERR "new volume ID is '$volid'\n";
5659             $d->{volid} = $volid;
5660             my $path = PVE::Storage::path($cfg, $volid);
5661
5662             PVE::Storage::activate_volumes($cfg,[$volid]);
5663
5664             my $write_zeros = 1;
5665             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5666                 $write_zeros = 0;
5667             }
5668
5669             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5670
5671             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5672             $map->{$virtdev} = $volid;
5673         }
5674
5675         $fh->seek(0, 0) || die "seek failed - $!\n";
5676
5677         my $outfd = new IO::File ($tmpfn, "w") ||
5678             die "unable to write config for VM $vmid\n";
5679
5680         my $cookie = { netcount => 0 };
5681         while (defined(my $line = <$fh>)) {
5682             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5683         }
5684
5685         $fh->close();
5686         $outfd->close();
5687     };
5688
5689     eval {
5690         # enable interrupts
5691         local $SIG{INT} =
5692             local $SIG{TERM} =
5693             local $SIG{QUIT} =
5694             local $SIG{HUP} =
5695             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5696         local $SIG{ALRM} = sub { die "got timeout\n"; };
5697
5698         $oldtimeout = alarm($timeout);
5699
5700         my $parser = sub {
5701             my $line = shift;
5702
5703             print "$line\n";
5704
5705             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5706                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5707                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5708             } elsif ($line =~ m/^CTIME: /) {
5709                 # we correctly received the vma config, so we can disable
5710                 # the timeout now for disk allocation (set to 10 minutes, so
5711                 # that we always timeout if something goes wrong)
5712                 alarm(600);
5713                 &$print_devmap();
5714                 print $fifofh "done\n";
5715                 my $tmp = $oldtimeout || 0;
5716                 $oldtimeout = undef;
5717                 alarm($tmp);
5718                 close($fifofh);
5719             }
5720         };
5721
5722         print "restore vma archive: $cmd\n";
5723         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5724     };
5725     my $err = $@;
5726
5727     alarm($oldtimeout) if $oldtimeout;
5728
5729     my $vollist = [];
5730     foreach my $devname (keys %$devinfo) {
5731         my $volid = $devinfo->{$devname}->{volid};
5732         push @$vollist, $volid if $volid;
5733     }
5734
5735     my $cfg = PVE::Storage::config();
5736     PVE::Storage::deactivate_volumes($cfg, $vollist);
5737
5738     unlink $mapfifo;
5739
5740     if ($err) {
5741         rmtree $tmpdir;
5742         unlink $tmpfn;
5743
5744         foreach my $devname (keys %$devinfo) {
5745             my $volid = $devinfo->{$devname}->{volid};
5746             next if !$volid;
5747             eval {
5748                 if ($volid =~ m|^/|) {
5749                     unlink $volid || die 'unlink failed\n';
5750                 } else {
5751                     PVE::Storage::vdisk_free($cfg, $volid);
5752                 }
5753                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5754             };
5755             print STDERR "unable to cleanup '$volid' - $@" if $@;
5756         }
5757         die $err;
5758     }
5759
5760     rmtree $tmpdir;
5761
5762     rename($tmpfn, $conffile) ||
5763         die "unable to commit configuration file '$conffile'\n";
5764
5765     PVE::Cluster::cfs_update(); # make sure we read new file
5766
5767     eval { rescan($vmid, 1); };
5768     warn $@ if $@;
5769 }
5770
5771 sub restore_tar_archive {
5772     my ($archive, $vmid, $user, $opts) = @_;
5773
5774     if ($archive ne '-') {
5775         my $firstfile = tar_archive_read_firstfile($archive);
5776         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5777             if $firstfile ne 'qemu-server.conf';
5778     }
5779
5780     my $storecfg = PVE::Storage::config();
5781
5782     # destroy existing data - keep empty config
5783     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5784     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5785
5786     my $tocmd = "/usr/lib/qemu-server/qmextract";
5787
5788     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5789     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5790     $tocmd .= ' --prealloc' if $opts->{prealloc};
5791     $tocmd .= ' --info' if $opts->{info};
5792
5793     # tar option "xf" does not autodetect compression when read from STDIN,
5794     # so we pipe to zcat
5795     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5796         PVE::Tools::shellquote("--to-command=$tocmd");
5797
5798     my $tmpdir = "/var/tmp/vzdumptmp$$";
5799     mkpath $tmpdir;
5800
5801     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5802     local $ENV{VZDUMP_VMID} = $vmid;
5803     local $ENV{VZDUMP_USER} = $user;
5804
5805     my $conffile = PVE::QemuConfig->config_file($vmid);
5806     my $tmpfn = "$conffile.$$.tmp";
5807
5808     # disable interrupts (always do cleanups)
5809     local $SIG{INT} =
5810         local $SIG{TERM} =
5811         local $SIG{QUIT} =
5812         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5813
5814     eval {
5815         # enable interrupts
5816         local $SIG{INT} =
5817             local $SIG{TERM} =
5818             local $SIG{QUIT} =
5819             local $SIG{HUP} =
5820             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5821
5822         if ($archive eq '-') {
5823             print "extracting archive from STDIN\n";
5824             run_command($cmd, input => "<&STDIN");
5825         } else {
5826             print "extracting archive '$archive'\n";
5827             run_command($cmd);
5828         }
5829
5830         return if $opts->{info};
5831
5832         # read new mapping
5833         my $map = {};
5834         my $statfile = "$tmpdir/qmrestore.stat";
5835         if (my $fd = IO::File->new($statfile, "r")) {
5836             while (defined (my $line = <$fd>)) {
5837                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5838                     $map->{$1} = $2 if $1;
5839                 } else {
5840                     print STDERR "unable to parse line in statfile - $line\n";
5841                 }
5842             }
5843             $fd->close();
5844         }
5845
5846         my $confsrc = "$tmpdir/qemu-server.conf";
5847
5848         my $srcfd = new IO::File($confsrc, "r") ||
5849             die "unable to open file '$confsrc'\n";
5850
5851         my $outfd = new IO::File ($tmpfn, "w") ||
5852             die "unable to write config for VM $vmid\n";
5853
5854         my $cookie = { netcount => 0 };
5855         while (defined (my $line = <$srcfd>)) {
5856             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5857         }
5858
5859         $srcfd->close();
5860         $outfd->close();
5861     };
5862     my $err = $@;
5863
5864     if ($err) {
5865
5866         unlink $tmpfn;
5867
5868         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5869
5870         die $err;
5871     }
5872
5873     rmtree $tmpdir;
5874
5875     rename $tmpfn, $conffile ||
5876         die "unable to commit configuration file '$conffile'\n";
5877
5878     PVE::Cluster::cfs_update(); # make sure we read new file
5879
5880     eval { rescan($vmid, 1); };
5881     warn $@ if $@;
5882 };
5883
5884 sub foreach_storage_used_by_vm {
5885     my ($conf, $func) = @_;
5886
5887     my $sidhash = {};
5888
5889     foreach_drive($conf, sub {
5890         my ($ds, $drive) = @_;
5891         return if drive_is_cdrom($drive);
5892
5893         my $volid = $drive->{file};
5894
5895         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5896         $sidhash->{$sid} = $sid if $sid;
5897     });
5898
5899     foreach my $sid (sort keys %$sidhash) {
5900         &$func($sid);
5901     }
5902 }
5903
5904 sub do_snapshots_with_qemu {
5905     my ($storecfg, $volid) = @_;
5906
5907     my $storage_name = PVE::Storage::parse_volume_id($volid);
5908
5909     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5910         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5911         return 1;
5912     }
5913
5914     if ($volid =~ m/\.(qcow2|qed)$/){
5915         return 1;
5916     }
5917
5918     return undef;
5919 }
5920
5921 sub qga_check_running {
5922     my ($vmid) = @_;
5923
5924     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5925     if ($@) {
5926         warn "Qemu Guest Agent is not running - $@";
5927         return 0;
5928     }
5929     return 1;
5930 }
5931
5932 sub template_create {
5933     my ($vmid, $conf, $disk) = @_;
5934
5935     my $storecfg = PVE::Storage::config();
5936
5937     foreach_drive($conf, sub {
5938         my ($ds, $drive) = @_;
5939
5940         return if drive_is_cdrom($drive);
5941         return if $disk && $ds ne $disk;
5942
5943         my $volid = $drive->{file};
5944         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5945
5946         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5947         $drive->{file} = $voliddst;
5948         $conf->{$ds} = print_drive($vmid, $drive);
5949         PVE::QemuConfig->write_config($vmid, $conf);
5950     });
5951 }
5952
5953 sub qemu_img_convert {
5954     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5955
5956     my $storecfg = PVE::Storage::config();
5957     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5958     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5959
5960     if ($src_storeid && $dst_storeid) {
5961
5962         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5963
5964         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5965         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5966
5967         my $src_format = qemu_img_format($src_scfg, $src_volname);
5968         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5969
5970         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5971         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5972
5973         my $cmd = [];
5974         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5975         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5976         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5977         if ($is_zero_initialized) {
5978             push @$cmd, "zeroinit:$dst_path";
5979         } else {
5980             push @$cmd, $dst_path;
5981         }
5982
5983         my $parser = sub {
5984             my $line = shift;
5985             if($line =~ m/\((\S+)\/100\%\)/){
5986                 my $percent = $1;
5987                 my $transferred = int($size * $percent / 100);
5988                 my $remaining = $size - $transferred;
5989
5990                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5991             }
5992
5993         };
5994
5995         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5996         my $err = $@;
5997         die "copy failed: $err" if $err;
5998     }
5999 }
6000
6001 sub qemu_img_format {
6002     my ($scfg, $volname) = @_;
6003
6004     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6005         return $1;
6006     } else {
6007         return "raw";
6008     }
6009 }
6010
6011 sub qemu_drive_mirror {
6012     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6013
6014     $jobs = {} if !$jobs;
6015
6016     my $qemu_target;
6017     my $format;
6018     $jobs->{"drive-$drive"} = {};
6019
6020     if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
6021         my $server = $1;
6022         my $port = $2;
6023         my $exportname = $3;
6024
6025         $format = "nbd";
6026         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
6027         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
6028         my $cmd = ['socat', '-T30', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=5"];
6029
6030         my $pid = fork();
6031         if (!defined($pid)) {
6032             die "forking socat tunnel failed\n";
6033         } elsif ($pid == 0) {
6034             exec(@$cmd);
6035             warn "exec failed: $!\n";
6036             POSIX::_exit(-1);
6037         }
6038         $jobs->{"drive-$drive"}->{pid} = $pid;
6039
6040         my $timeout = 0;
6041         while (!-S $unixsocket) {
6042             die "nbd connection helper timed out\n"
6043                 if $timeout++ > 5;
6044             sleep 1;
6045         }
6046     } else {
6047         my $storecfg = PVE::Storage::config();
6048         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6049
6050         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6051
6052         $format = qemu_img_format($dst_scfg, $dst_volname);
6053
6054         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6055
6056         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6057     }
6058
6059     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6060     $opts->{format} = $format if $format;
6061
6062     print "drive mirror is starting for drive-$drive\n";
6063
6064     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6065
6066     if (my $err = $@) {
6067         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6068         die "mirroring error: $err";
6069     }
6070
6071     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6072 }
6073
6074 sub qemu_drive_mirror_monitor {
6075     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6076
6077     eval {
6078         my $err_complete = 0;
6079
6080         while (1) {
6081             die "storage migration timed out\n" if $err_complete > 300;
6082
6083             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6084
6085             my $running_mirror_jobs = {};
6086             foreach my $stat (@$stats) {
6087                 next if $stat->{type} ne 'mirror';
6088                 $running_mirror_jobs->{$stat->{device}} = $stat;
6089             }
6090
6091             my $readycounter = 0;
6092
6093             foreach my $job (keys %$jobs) {
6094
6095                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6096                     print "$job : finished\n";
6097                     delete $jobs->{$job};
6098                     next;
6099                 }
6100
6101                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6102
6103                 my $busy = $running_mirror_jobs->{$job}->{busy};
6104                 my $ready = $running_mirror_jobs->{$job}->{ready};
6105                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6106                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6107                     my $remaining = $total - $transferred;
6108                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6109
6110                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6111                 }
6112
6113                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6114             }
6115
6116             last if scalar(keys %$jobs) == 0;
6117
6118             if ($readycounter == scalar(keys %$jobs)) {
6119                 print "all mirroring jobs are ready \n";
6120                 last if $skipcomplete; #do the complete later
6121
6122                 if ($vmiddst && $vmiddst != $vmid) {
6123                     my $agent_running = $qga && qga_check_running($vmid);
6124                     if ($agent_running) {
6125                         print "freeze filesystem\n";
6126                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6127                     } else {
6128                         print "suspend vm\n";
6129                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6130                     }
6131
6132                     # if we clone a disk for a new target vm, we don't switch the disk
6133                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6134
6135                     if ($agent_running) {
6136                         print "unfreeze filesystem\n";
6137                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6138                     } else {
6139                         print "resume vm\n";
6140                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6141                     }
6142
6143                     last;
6144                 } else {
6145
6146                     foreach my $job (keys %$jobs) {
6147                         # try to switch the disk if source and destination are on the same guest
6148                         print "$job: Completing block job...\n";
6149
6150                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6151                         if ($@ =~ m/cannot be completed/) {
6152                             print "$job: Block job cannot be completed, try again.\n";
6153                             $err_complete++;
6154                         }else {
6155                             print "$job: Completed successfully.\n";
6156                             $jobs->{$job}->{complete} = 1;
6157                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6158                         }
6159                     }
6160                 }
6161             }
6162             sleep 1;
6163         }
6164     };
6165     my $err = $@;
6166
6167     if ($err) {
6168         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6169         die "mirroring error: $err";
6170     }
6171
6172 }
6173
6174 sub qemu_blockjobs_cancel {
6175     my ($vmid, $jobs) = @_;
6176
6177     foreach my $job (keys %$jobs) {
6178         print "$job: Cancelling block job\n";
6179         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6180         $jobs->{$job}->{cancel} = 1;
6181     }
6182
6183     while (1) {
6184         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6185
6186         my $running_jobs = {};
6187         foreach my $stat (@$stats) {
6188             $running_jobs->{$stat->{device}} = $stat;
6189         }
6190
6191         foreach my $job (keys %$jobs) {
6192
6193             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6194                 print "$job: Done.\n";
6195                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6196                 delete $jobs->{$job};
6197             }
6198         }
6199
6200         last if scalar(keys %$jobs) == 0;
6201
6202         sleep 1;
6203     }
6204 }
6205
6206 sub qemu_blockjobs_finish_tunnel {
6207    my ($vmid, $job, $cpid) = @_;
6208
6209    return if !$cpid;
6210
6211    for (my $i = 1; $i < 20; $i++) {
6212         my $waitpid = waitpid($cpid, WNOHANG);
6213         last if (defined($waitpid) && ($waitpid == $cpid));
6214
6215         if ($i == 10) {
6216             kill(15, $cpid);
6217         } elsif ($i >= 15) {
6218             kill(9, $cpid);
6219         }
6220         sleep (1);
6221     }
6222     unlink "/run/qemu-server/$vmid.mirror-$job";
6223 }
6224
6225 sub clone_disk {
6226     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6227         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6228
6229     my $newvolid;
6230
6231     if (!$full) {
6232         print "create linked clone of drive $drivename ($drive->{file})\n";
6233         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6234         push @$newvollist, $newvolid;
6235     } else {
6236
6237         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6238         $storeid = $storage if $storage;
6239
6240         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6241         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6242
6243         print "create full clone of drive $drivename ($drive->{file})\n";
6244         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, undef, ($size/1024));
6245         push @$newvollist, $newvolid;
6246
6247         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6248
6249         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6250         if (!$running || $snapname) {
6251             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6252         } else {
6253
6254             my $kvmver = get_running_qemu_version ($vmid);
6255             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6256                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6257                     if $drive->{iothread};
6258             }
6259
6260             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6261         }
6262     }
6263
6264     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6265
6266     my $disk = $drive;
6267     $disk->{format} = undef;
6268     $disk->{file} = $newvolid;
6269     $disk->{size} = $size;
6270
6271     return $disk;
6272 }
6273
6274 # this only works if VM is running
6275 sub get_current_qemu_machine {
6276     my ($vmid) = @_;
6277
6278     my $cmd = { execute => 'query-machines', arguments => {} };
6279     my $res = vm_qmp_command($vmid, $cmd);
6280
6281     my ($current, $default);
6282     foreach my $e (@$res) {
6283         $default = $e->{name} if $e->{'is-default'};
6284         $current = $e->{name} if $e->{'is-current'};
6285     }
6286
6287     # fallback to the default machine if current is not supported by qemu
6288     return $current || $default || 'pc';
6289 }
6290
6291 sub get_running_qemu_version {
6292     my ($vmid) = @_;
6293     my $cmd = { execute => 'query-version', arguments => {} };
6294     my $res = vm_qmp_command($vmid, $cmd);
6295     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6296 }
6297
6298 sub qemu_machine_feature_enabled {
6299     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6300
6301     my $current_major;
6302     my $current_minor;
6303
6304     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6305
6306         $current_major = $3;
6307         $current_minor = $4;
6308
6309     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6310
6311         $current_major = $1;
6312         $current_minor = $2;
6313     }
6314
6315     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6316
6317
6318 }
6319
6320 sub qemu_machine_pxe {
6321     my ($vmid, $conf, $machine) = @_;
6322
6323     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6324
6325     foreach my $opt (keys %$conf) {
6326         next if $opt !~ m/^net(\d+)$/;
6327         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6328         next if !$net;
6329         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6330         return $machine.".pxe" if $romfile =~ m/pxe/;
6331         last;
6332     }
6333
6334     return $machine;
6335 }
6336
6337 sub qemu_use_old_bios_files {
6338     my ($machine_type) = @_;
6339
6340     return if !$machine_type;
6341
6342     my $use_old_bios_files = undef;
6343
6344     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6345         $machine_type = $1;
6346         $use_old_bios_files = 1;
6347     } else {
6348         my $kvmver = kvm_user_version();
6349         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6350         # load new efi bios files on migration. So this hack is required to allow
6351         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6352         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6353         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6354     }
6355
6356     return ($use_old_bios_files, $machine_type);
6357 }
6358
6359 sub create_efidisk {
6360     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6361
6362     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6363
6364     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6365     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6366     PVE::Storage::activate_volumes($storecfg, [$volid]);
6367
6368     my $path = PVE::Storage::path($storecfg, $volid);
6369     eval {
6370         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6371     };
6372     die "Copying EFI vars image failed: $@" if $@;
6373
6374     return ($volid, $vars_size);
6375 }
6376
6377 sub lspci {
6378
6379     my $devices = {};
6380
6381     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6382             my (undef, $id, $function) = @_;
6383             my $res = { id => $id, function => $function};
6384             push @{$devices->{$id}}, $res;
6385     });
6386
6387     # Entries should be sorted by functions.
6388     foreach my $id (keys %$devices) {
6389         my $dev = $devices->{$id};
6390         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6391     }
6392
6393     return $devices;
6394 }
6395
6396 sub vm_iothreads_list {
6397     my ($vmid) = @_;
6398
6399     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6400
6401     my $iothreads = {};
6402     foreach my $iothread (@$res) {
6403         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6404     }
6405
6406     return $iothreads;
6407 }
6408
6409 sub scsihw_infos {
6410     my ($conf, $drive) = @_;
6411
6412     my $maxdev = 0;
6413
6414     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6415         $maxdev = 7;
6416     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6417         $maxdev = 1;
6418     } else {
6419         $maxdev = 256;
6420     }
6421
6422     my $controller = int($drive->{index} / $maxdev);
6423     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6424
6425     return ($maxdev, $controller, $controller_prefix);
6426 }
6427
6428 sub add_hyperv_enlightenments {
6429     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6430
6431     return if $winversion < 6;
6432     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6433
6434     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6435
6436     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6437         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6438         push @$cpuFlags , 'hv_vapic';
6439         push @$cpuFlags , 'hv_time';
6440     } else {
6441         push @$cpuFlags , 'hv_spinlocks=0xffff';
6442     }
6443
6444     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6445         push @$cpuFlags , 'hv_reset';
6446         push @$cpuFlags , 'hv_vpindex';
6447         push @$cpuFlags , 'hv_runtime';
6448     }
6449
6450     if ($winversion >= 7) {
6451         push @$cpuFlags , 'hv_relaxed';
6452     }
6453 }
6454
6455 sub windows_version {
6456     my ($ostype) = @_;
6457
6458     return 0 if !$ostype;
6459
6460     my $winversion = 0;
6461
6462     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6463         $winversion = 5;
6464     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6465         $winversion = 6;
6466     } elsif ($ostype =~ m/^win(\d+)$/) {
6467         $winversion = $1;
6468     }
6469
6470     return $winversion;
6471 }
6472
6473 sub resolve_dst_disk_format {
6474         my ($storecfg, $storeid, $src_volname, $format) = @_;
6475         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6476
6477         if (!$format) {
6478             # if no target format is specified, use the source disk format as hint
6479             if ($src_volname) {
6480                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6481                 $format = qemu_img_format($scfg, $src_volname);
6482             } else {
6483                 return $defFormat;
6484             }
6485         }
6486
6487         # test if requested format is supported - else use default
6488         my $supported = grep { $_ eq $format } @$validFormats;
6489         $format = $defFormat if !$supported;
6490         return $format;
6491 }
6492
6493 sub resolve_first_disk {
6494     my $conf = shift;
6495     my @disks = PVE::QemuServer::valid_drive_names();
6496     my $firstdisk;
6497     foreach my $ds (reverse @disks) {
6498         next if !$conf->{$ds};
6499         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6500         next if PVE::QemuServer::drive_is_cdrom($disk);
6501         $firstdisk = $ds;
6502     }
6503     return $firstdisk;
6504 }
6505
6506 sub generate_smbios1_uuid {
6507     my ($uuid, $uuid_str);
6508     UUID::generate($uuid);
6509     UUID::unparse($uuid, $uuid_str);
6510     return "uuid=$uuid_str";
6511 }
6512
6513 # bash completion helper
6514
6515 sub complete_backup_archives {
6516     my ($cmdname, $pname, $cvalue) = @_;
6517
6518     my $cfg = PVE::Storage::config();
6519
6520     my $storeid;
6521
6522     if ($cvalue =~ m/^([^:]+):/) {
6523         $storeid = $1;
6524     }
6525
6526     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6527
6528     my $res = [];
6529     foreach my $id (keys %$data) {
6530         foreach my $item (@{$data->{$id}}) {
6531             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6532             push @$res, $item->{volid} if defined($item->{volid});
6533         }
6534     }
6535
6536     return $res;
6537 }
6538
6539 my $complete_vmid_full = sub {
6540     my ($running) = @_;
6541
6542     my $idlist = vmstatus();
6543
6544     my $res = [];
6545
6546     foreach my $id (keys %$idlist) {
6547         my $d = $idlist->{$id};
6548         if (defined($running)) {
6549             next if $d->{template};
6550             next if $running && $d->{status} ne 'running';
6551             next if !$running && $d->{status} eq 'running';
6552         }
6553         push @$res, $id;
6554
6555     }
6556     return $res;
6557 };
6558
6559 sub complete_vmid {
6560     return &$complete_vmid_full();
6561 }
6562
6563 sub complete_vmid_stopped {
6564     return &$complete_vmid_full(0);
6565 }
6566
6567 sub complete_vmid_running {
6568     return &$complete_vmid_full(1);
6569 }
6570
6571 sub complete_storage {
6572
6573     my $cfg = PVE::Storage::config();
6574     my $ids = $cfg->{ids};
6575
6576     my $res = [];
6577     foreach my $sid (keys %$ids) {
6578         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6579         next if !$ids->{$sid}->{content}->{images};
6580         push @$res, $sid;
6581     }
6582
6583     return $res;
6584 }
6585
6586 sub nbd_stop {
6587     my ($vmid) = @_;
6588
6589     vm_mon_cmd($vmid, 'nbd-server-stop');
6590 }
6591
6592 1;