PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42
  43 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  44
  45 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  46
  47 # Note about locking: we use flock on the config file protect
  48 # against concurent actions.
  49 # Aditionaly, we have a 'lock' setting in the config file. This
  50 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  51 # allowed when such lock is set. But you can ignore this kind of
  52 # lock with the --skiplock flag.
  53
  54 cfs_register_file('/qemu-server/',
  55                   \&parse_vm_config,
  56                   \&write_vm_config);
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  72     type => 'string',
  73     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  74     description => "The drive's backing file's data format.",
  75     optional => 1,
  76 });
  77
  78 #no warnings 'redefine';
  79
  80 sub cgroups_write {
  81    my ($controller, $vmid, $option, $value) = @_;
  82
  83    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  84    PVE::ProcFSTools::write_proc_entry($path, $value);
  85
  86 }
  87
  88 my $nodename = PVE::INotify::nodename();
  89
  90 mkdir "/etc/pve/nodes/$nodename";
  91 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  92 mkdir $confdir;
  93
  94 my $var_run_tmpdir = "/var/run/qemu-server";
  95 mkdir $var_run_tmpdir;
  96
  97 my $lock_dir = "/var/lock/qemu-server";
  98 mkdir $lock_dir;
  99
 100 my $pcisysfs = "/sys/bus/pci";
 101
 102 my $cpu_vendor_list = {
 103     # Intel CPUs
 104     486 => 'GenuineIntel',
 105     pentium => 'GenuineIntel',
 106     pentium2  => 'GenuineIntel',
 107     pentium3  => 'GenuineIntel',
 108     coreduo => 'GenuineIntel',
 109     core2duo => 'GenuineIntel',
 110     Conroe  => 'GenuineIntel',
 111     Penryn  => 'GenuineIntel',
 112     Nehalem  => 'GenuineIntel',
 113     Westmere => 'GenuineIntel',
 114     SandyBridge => 'GenuineIntel',
 115     IvyBridge => 'GenuineIntel',
 116     Haswell => 'GenuineIntel',
 117     'Haswell-noTSX' => 'GenuineIntel',
 118     Broadwell => 'GenuineIntel',
 119     'Broadwell-noTSX' => 'GenuineIntel',
 120     'Skylake-Client' => 'GenuineIntel',
 121
 122     # AMD CPUs
 123     athlon => 'AuthenticAMD',
 124     phenom  => 'AuthenticAMD',
 125     Opteron_G1  => 'AuthenticAMD',
 126     Opteron_G2  => 'AuthenticAMD',
 127     Opteron_G3  => 'AuthenticAMD',
 128     Opteron_G4  => 'AuthenticAMD',
 129     Opteron_G5  => 'AuthenticAMD',
 130
 131     # generic types, use vendor from host node
 132     host => 'default',
 133     kvm32 => 'default',
 134     kvm64 => 'default',
 135     qemu32 => 'default',
 136     qemu64 => 'default',
 137 };
 138
 139 my $cpu_fmt = {
 140     cputype => {
 141         description => "Emulated CPU type.",
 142         type => 'string',
 143         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 144         default => 'kvm64',
 145         default_key => 1,
 146     },
 147     hidden => {
 148         description => "Do not identify as a KVM virtual machine.",
 149         type => 'boolean',
 150         optional => 1,
 151         default => 0
 152     },
 153     flags => {
 154         description => "Override CPU flags. Currently only the 'pcid' flag is supported."
 155                      . " Use '+pcid' or '-pcid' to enable or disable."
 156                      . " This takes precedence over flags coming from the cpu type or changed implicitly via the OS type.",
 157         format_description => 'flaglist',
 158         type => 'string',
 159         pattern => '[+-]pcid',
 160         optional => 1,
 161         default => '',
 162     },
 163 };
 164
 165 my $watchdog_fmt = {
 166     model => {
 167         default_key => 1,
 168         type => 'string',
 169         enum => [qw(i6300esb ib700)],
 170         description => "Watchdog type to emulate.",
 171         default => 'i6300esb',
 172         optional => 1,
 173     },
 174     action => {
 175         type => 'string',
 176         enum => [qw(reset shutdown poweroff pause debug none)],
 177         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 178         optional => 1,
 179     },
 180 };
 181 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 182
 183 my $confdesc = {
 184     onboot => {
 185         optional => 1,
 186         type => 'boolean',
 187         description => "Specifies whether a VM will be started during system bootup.",
 188         default => 0,
 189     },
 190     autostart => {
 191         optional => 1,
 192         type => 'boolean',
 193         description => "Automatic restart after crash (currently ignored).",
 194         default => 0,
 195     },
 196     hotplug => {
 197         optional => 1,
 198         type => 'string', format => 'pve-hotplug-features',
 199         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 200         default => 'network,disk,usb',
 201     },
 202     reboot => {
 203         optional => 1,
 204         type => 'boolean',
 205         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 206         default => 1,
 207     },
 208     lock => {
 209         optional => 1,
 210         type => 'string',
 211         description => "Lock/unlock the VM.",
 212         enum => [qw(migrate backup snapshot rollback)],
 213     },
 214     cpulimit => {
 215         optional => 1,
 216         type => 'number',
 217         description => "Limit of CPU usage.",
 218         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 219         minimum => 0,
 220         maximum => 128,
 221         default => 0,
 222     },
 223     cpuunits => {
 224         optional => 1,
 225         type => 'integer',
 226         description => "CPU weight for a VM.",
 227         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 228         minimum => 2,
 229         maximum => 262144,
 230         default => 1024,
 231     },
 232     memory => {
 233         optional => 1,
 234         type => 'integer',
 235         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 236         minimum => 16,
 237         default => 512,
 238     },
 239     balloon => {
 240         optional => 1,
 241         type => 'integer',
 242         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 243         minimum => 0,
 244     },
 245     shares => {
 246         optional => 1,
 247         type => 'integer',
 248         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 249         minimum => 0,
 250         maximum => 50000,
 251         default => 1000,
 252     },
 253     keyboard => {
 254         optional => 1,
 255         type => 'string',
 256         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.".
 257                        "It should not be necessary to set it.",
 258         enum => PVE::Tools::kvmkeymaplist(),
 259         default => undef,
 260     },
 261     name => {
 262         optional => 1,
 263         type => 'string', format => 'dns-name',
 264         description => "Set a name for the VM. Only used on the configuration web interface.",
 265     },
 266     scsihw => {
 267         optional => 1,
 268         type => 'string',
 269         description => "SCSI controller model",
 270         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 271         default => 'lsi',
 272     },
 273     description => {
 274         optional => 1,
 275         type => 'string',
 276         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 277     },
 278     ostype => {
 279         optional => 1,
 280         type => 'string',
 281         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 282         description => "Specify guest operating system.",
 283         verbose_description => <<EODESC,
 284 Specify guest operating system. This is used to enable special
 285 optimization/features for specific operating systems:
 286
 287 [horizontal]
 288 other;; unspecified OS
 289 wxp;; Microsoft Windows XP
 290 w2k;; Microsoft Windows 2000
 291 w2k3;; Microsoft Windows 2003
 292 w2k8;; Microsoft Windows 2008
 293 wvista;; Microsoft Windows Vista
 294 win7;; Microsoft Windows 7
 295 win8;; Microsoft Windows 8/2012/2012r2
 296 win10;; Microsoft Windows 10/2016
 297 l24;; Linux 2.4 Kernel
 298 l26;; Linux 2.6/3.X Kernel
 299 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 300 EODESC
 301     },
 302     boot => {
 303         optional => 1,
 304         type => 'string',
 305         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 306         pattern => '[acdn]{1,4}',
 307         default => 'cdn',
 308     },
 309     bootdisk => {
 310         optional => 1,
 311         type => 'string', format => 'pve-qm-bootdisk',
 312         description => "Enable booting from specified disk.",
 313         pattern => '(ide|sata|scsi|virtio)\d+',
 314     },
 315     smp => {
 316         optional => 1,
 317         type => 'integer',
 318         description => "The number of CPUs. Please use option -sockets instead.",
 319         minimum => 1,
 320         default => 1,
 321     },
 322     sockets => {
 323         optional => 1,
 324         type => 'integer',
 325         description => "The number of CPU sockets.",
 326         minimum => 1,
 327         default => 1,
 328     },
 329     cores => {
 330         optional => 1,
 331         type => 'integer',
 332         description => "The number of cores per socket.",
 333         minimum => 1,
 334         default => 1,
 335     },
 336     numa => {
 337         optional => 1,
 338         type => 'boolean',
 339         description => "Enable/disable NUMA.",
 340         default => 0,
 341     },
 342     hugepages => {
 343         optional => 1,
 344         type => 'string',
 345         description => "Enable/disable hugepages memory.",
 346         enum => [qw(any 2 1024)],
 347     },
 348     vcpus => {
 349         optional => 1,
 350         type => 'integer',
 351         description => "Number of hotplugged vcpus.",
 352         minimum => 1,
 353         default => 0,
 354     },
 355     acpi => {
 356         optional => 1,
 357         type => 'boolean',
 358         description => "Enable/disable ACPI.",
 359         default => 1,
 360     },
 361     agent => {
 362         optional => 1,
 363         type => 'boolean',
 364         description => "Enable/disable Qemu GuestAgent.",
 365         default => 0,
 366     },
 367     kvm => {
 368         optional => 1,
 369         type => 'boolean',
 370         description => "Enable/disable KVM hardware virtualization.",
 371         default => 1,
 372     },
 373     tdf => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Enable/disable time drift fix.",
 377         default => 0,
 378     },
 379     localtime => {
 380         optional => 1,
 381         type => 'boolean',
 382         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 383     },
 384     freeze => {
 385         optional => 1,
 386         type => 'boolean',
 387         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 388     },
 389     vga => {
 390         optional => 1,
 391         type => 'string',
 392         description => "Select the VGA type.",
 393         verbose_description => "Select the VGA type. If you want to use high resolution" .
 394             " modes (>= 1280x1024x16) then you should use the options " .
 395             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 396             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 397             "display sever. For win* OS you can select how many independent " .
 398             "displays you want, Linux guests can add displays them self. " .
 399             "You can also run without any graphic card, using a serial device" .
 400             " as terminal.",
 401         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 402     },
 403     watchdog => {
 404         optional => 1,
 405         type => 'string', format => 'pve-qm-watchdog',
 406         description => "Create a virtual hardware watchdog device.",
 407         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 408             " (by a guest action), the watchdog must be periodically polled " .
 409             "by an agent inside the guest or else the watchdog will reset " .
 410             "the guest (or execute the respective action specified)",
 411     },
 412     startdate => {
 413         optional => 1,
 414         type => 'string',
 415         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 416         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 417         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 418         default => 'now',
 419     },
 420     startup =>  get_standard_option('pve-startup-order'),
 421     template => {
 422         optional => 1,
 423         type => 'boolean',
 424         description => "Enable/disable Template.",
 425         default => 0,
 426     },
 427     args => {
 428         optional => 1,
 429         type => 'string',
 430         description => "Arbitrary arguments passed to kvm.",
 431         verbose_description => <<EODESCR,
 432 Arbitrary arguments passed to kvm, for example:
 433
 434 args: -no-reboot -no-hpet
 435
 436 NOTE: this option is for experts only.
 437 EODESCR
 438     },
 439     tablet => {
 440         optional => 1,
 441         type => 'boolean',
 442         default => 1,
 443         description => "Enable/disable the USB tablet device.",
 444         verbose_description => "Enable/disable the USB tablet device. This device is " .
 445             "usually needed to allow absolute mouse positioning with VNC. " .
 446             "Else the mouse runs out of sync with normal VNC clients. " .
 447             "If you're running lots of console-only guests on one host, " .
 448             "you may consider disabling this to save some context switches. " .
 449             "This is turned off by default if you use spice (-vga=qxl).",
 450     },
 451     migrate_speed => {
 452         optional => 1,
 453         type => 'integer',
 454         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 455         minimum => 0,
 456         default => 0,
 457     },
 458     migrate_downtime => {
 459         optional => 1,
 460         type => 'number',
 461         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 462         minimum => 0,
 463         default => 0.1,
 464     },
 465     cdrom => {
 466         optional => 1,
 467         type => 'string', format => 'pve-qm-ide',
 468         typetext => '<volume>',
 469         description => "This is an alias for option -ide2",
 470     },
 471     cpu => {
 472         optional => 1,
 473         description => "Emulated CPU type.",
 474         type => 'string',
 475         format => $cpu_fmt,
 476     },
 477     parent => get_standard_option('pve-snapshot-name', {
 478         optional => 1,
 479         description => "Parent snapshot name. This is used internally, and should not be modified.",
 480     }),
 481     snaptime => {
 482         optional => 1,
 483         description => "Timestamp for snapshots.",
 484         type => 'integer',
 485         minimum => 0,
 486     },
 487     vmstate => {
 488         optional => 1,
 489         type => 'string', format => 'pve-volume-id',
 490         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 491     },
 492     vmstatestorage => get_standard_option('pve-storage-id', {
 493         description => "Default storage for VM state volumes/files.",
 494         optional => 1,
 495     }),
 496     machine => {
 497         description => "Specific the Qemu machine type.",
 498         type => 'string',
 499         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 500         maxLength => 40,
 501         optional => 1,
 502     },
 503     smbios1 => {
 504         description => "Specify SMBIOS type 1 fields.",
 505         type => 'string', format => 'pve-qm-smbios1',
 506         maxLength => 256,
 507         optional => 1,
 508     },
 509     protection => {
 510         optional => 1,
 511         type => 'boolean',
 512         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 513         default => 0,
 514     },
 515     bios => {
 516         optional => 1,
 517         type => 'string',
 518         enum => [ qw(seabios ovmf) ],
 519         description => "Select BIOS implementation.",
 520         default => 'seabios',
 521     },
 522 };
 523
 524 # what about other qemu settings ?
 525 #cpu => 'string',
 526 #machine => 'string',
 527 #fda => 'file',
 528 #fdb => 'file',
 529 #mtdblock => 'file',
 530 #sd => 'file',
 531 #pflash => 'file',
 532 #snapshot => 'bool',
 533 #bootp => 'file',
 534 ##tftp => 'dir',
 535 ##smb => 'dir',
 536 #kernel => 'file',
 537 #append => 'string',
 538 #initrd => 'file',
 539 ##soundhw => 'string',
 540
 541 while (my ($k, $v) = each %$confdesc) {
 542     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 543 }
 544
 545 my $MAX_IDE_DISKS = 4;
 546 my $MAX_SCSI_DISKS = 14;
 547 my $MAX_VIRTIO_DISKS = 16;
 548 my $MAX_SATA_DISKS = 6;
 549 my $MAX_USB_DEVICES = 5;
 550 my $MAX_NETS = 32;
 551 my $MAX_UNUSED_DISKS = 8;
 552 my $MAX_HOSTPCI_DEVICES = 4;
 553 my $MAX_SERIAL_PORTS = 4;
 554 my $MAX_PARALLEL_PORTS = 3;
 555 my $MAX_NUMA = 8;
 556
 557 my $numa_fmt = {
 558     cpus => {
 559         type => "string",
 560         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 561         description => "CPUs accessing this NUMA node.",
 562         format_description => "id[-id];...",
 563     },
 564     memory => {
 565         type => "number",
 566         description => "Amount of memory this NUMA node provides.",
 567         optional => 1,
 568     },
 569     hostnodes => {
 570         type => "string",
 571         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 572         description => "Host NUMA nodes to use.",
 573         format_description => "id[-id];...",
 574         optional => 1,
 575     },
 576     policy => {
 577         type => 'string',
 578         enum => [qw(preferred bind interleave)],
 579         description => "NUMA allocation policy.",
 580         optional => 1,
 581     },
 582 };
 583 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 584 my $numadesc = {
 585     optional => 1,
 586     type => 'string', format => $numa_fmt,
 587     description => "NUMA topology.",
 588 };
 589 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 590
 591 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 592     $confdesc->{"numa$i"} = $numadesc;
 593 }
 594
 595 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 596                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 597                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 598 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 599
 600 my $net_fmt_bridge_descr = <<__EOD__;
 601 Bridge to attach the network device to. The Proxmox VE standard bridge
 602 is called 'vmbr0'.
 603
 604 If you do not specify a bridge, we create a kvm user (NATed) network
 605 device, which provides DHCP and DNS services. The following addresses
 606 are used:
 607
 608  10.0.2.2   Gateway
 609  10.0.2.3   DNS Server
 610  10.0.2.4   SMB Server
 611
 612 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 613 __EOD__
 614
 615 my $net_fmt = {
 616     macaddr => {
 617         type => 'string',
 618         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 619         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 620         format_description => "XX:XX:XX:XX:XX:XX",
 621         optional => 1,
 622     },
 623     model => {
 624         type => 'string',
 625         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 626         enum => $nic_model_list,
 627         default_key => 1,
 628     },
 629     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 630     bridge => {
 631         type => 'string',
 632         description => $net_fmt_bridge_descr,
 633         format_description => 'bridge',
 634         optional => 1,
 635     },
 636     queues => {
 637         type => 'integer',
 638         minimum => 0, maximum => 16,
 639         description => 'Number of packet queues to be used on the device.',
 640         optional => 1,
 641     },
 642     rate => {
 643         type => 'number',
 644         minimum => 0,
 645         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 646         optional => 1,
 647     },
 648     tag => {
 649         type => 'integer',
 650         minimum => 1, maximum => 4094,
 651         description => 'VLAN tag to apply to packets on this interface.',
 652         optional => 1,
 653     },
 654     trunks => {
 655         type => 'string',
 656         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 657         description => 'VLAN trunks to pass through this interface.',
 658         format_description => 'vlanid[;vlanid...]',
 659         optional => 1,
 660     },
 661     firewall => {
 662         type => 'boolean',
 663         description => 'Whether this interface should be protected by the firewall.',
 664         optional => 1,
 665     },
 666     link_down => {
 667         type => 'boolean',
 668         description => 'Whether this interface should be disconnected (like pulling the plug).',
 669         optional => 1,
 670     },
 671 };
 672
 673 my $netdesc = {
 674     optional => 1,
 675     type => 'string', format => $net_fmt,
 676     description => "Specify network devices.",
 677 };
 678
 679 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 680
 681 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 682     $confdesc->{"net$i"} = $netdesc;
 683 }
 684
 685 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 686 sub verify_volume_id_or_qm_path {
 687     my ($volid, $noerr) = @_;
 688
 689     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 690         return $volid;
 691     }
 692
 693     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 694     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 695     if ($@) {
 696         return undef if $noerr;
 697         die $@;
 698     }
 699     return $volid;
 700 }
 701
 702 my $drivename_hash;
 703
 704 my %drivedesc_base = (
 705     volume => { alias => 'file' },
 706     file => {
 707         type => 'string',
 708         format => 'pve-volume-id-or-qm-path',
 709         default_key => 1,
 710         format_description => 'volume',
 711         description => "The drive's backing volume.",
 712     },
 713     media => {
 714         type => 'string',
 715         enum => [qw(cdrom disk)],
 716         description => "The drive's media type.",
 717         default => 'disk',
 718         optional => 1
 719     },
 720     cyls => {
 721         type => 'integer',
 722         description => "Force the drive's physical geometry to have a specific cylinder count.",
 723         optional => 1
 724     },
 725     heads => {
 726         type => 'integer',
 727         description => "Force the drive's physical geometry to have a specific head count.",
 728         optional => 1
 729     },
 730     secs => {
 731         type => 'integer',
 732         description => "Force the drive's physical geometry to have a specific sector count.",
 733         optional => 1
 734     },
 735     trans => {
 736         type => 'string',
 737         enum => [qw(none lba auto)],
 738         description => "Force disk geometry bios translation mode.",
 739         optional => 1,
 740     },
 741     snapshot => {
 742         type => 'boolean',
 743         description => "Whether the drive should be included when making snapshots.",
 744         optional => 1,
 745     },
 746     cache => {
 747         type => 'string',
 748         enum => [qw(none writethrough writeback unsafe directsync)],
 749         description => "The drive's cache mode",
 750         optional => 1,
 751     },
 752     format => get_standard_option('pve-qm-image-format'),
 753     size => {
 754         type => 'string',
 755         format => 'disk-size',
 756         format_description => 'DiskSize',
 757         description => "Disk size. This is purely informational and has no effect.",
 758         optional => 1,
 759     },
 760     backup => {
 761         type => 'boolean',
 762         description => "Whether the drive should be included when making backups.",
 763         optional => 1,
 764     },
 765     replicate => {
 766         type => 'boolean',
 767         description => 'Whether the drive should considered for replication jobs.',
 768         optional => 1,
 769         default => 1,
 770     },
 771     rerror => {
 772         type => 'string',
 773         enum => [qw(ignore report stop)],
 774         description => 'Read error action.',
 775         optional => 1,
 776     },
 777     werror => {
 778         type => 'string',
 779         enum => [qw(enospc ignore report stop)],
 780         description => 'Write error action.',
 781         optional => 1,
 782     },
 783     aio => {
 784         type => 'string',
 785         enum => [qw(native threads)],
 786         description => 'AIO type to use.',
 787         optional => 1,
 788     },
 789     discard => {
 790         type => 'string',
 791         enum => [qw(ignore on)],
 792         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 793         optional => 1,
 794     },
 795     detect_zeroes => {
 796         type => 'boolean',
 797         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 798         optional => 1,
 799     },
 800     serial => {
 801         type => 'string',
 802         format => 'urlencoded',
 803         format_description => 'serial',
 804         maxLength => 20*3, # *3 since it's %xx url enoded
 805         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 806         optional => 1,
 807     }
 808 );
 809
 810 my %iothread_fmt = ( iothread => {
 811         type => 'boolean',
 812         description => "Whether to use iothreads for this drive",
 813         optional => 1,
 814 });
 815
 816 my %model_fmt = (
 817     model => {
 818         type => 'string',
 819         format => 'urlencoded',
 820         format_description => 'model',
 821         maxLength => 40*3, # *3 since it's %xx url enoded
 822         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 823         optional => 1,
 824     },
 825 );
 826
 827 my %queues_fmt = (
 828     queues => {
 829         type => 'integer',
 830         description => "Number of queues.",
 831         minimum => 2,
 832         optional => 1
 833     }
 834 );
 835
 836 my %scsiblock_fmt = (
 837     scsiblock => {
 838         type => 'boolean',
 839         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 840         optional => 1,
 841         default => 0,
 842     },
 843 );
 844
 845 my $add_throttle_desc = sub {
 846     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 847     my $d = {
 848         type => $type,
 849         format_description => $unit,
 850         description => "Maximum $what in $longunit.",
 851         optional => 1,
 852     };
 853     $d->{minimum} = $minimum if defined($minimum);
 854     $drivedesc_base{$key} = $d;
 855 };
 856 # throughput: (leaky bucket)
 857 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 858 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 859 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 860 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 861 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 862 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 863 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 864 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 865 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 866
 867 # pools: (pool of IO before throttling starts taking effect)
 868 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 869 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 870 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 871 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 872 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 873 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 874
 875 # burst lengths
 876 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 877 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 878 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 879 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 880 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 881 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 882
 883 # legacy support
 884 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
 885 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
 886 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
 887 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
 888
 889 my $ide_fmt = {
 890     %drivedesc_base,
 891     %model_fmt,
 892 };
 893 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 894
 895 my $idedesc = {
 896     optional => 1,
 897     type => 'string', format => $ide_fmt,
 898     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 899 };
 900 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 901
 902 my $scsi_fmt = {
 903     %drivedesc_base,
 904     %iothread_fmt,
 905     %queues_fmt,
 906     %scsiblock_fmt,
 907 };
 908 my $scsidesc = {
 909     optional => 1,
 910     type => 'string', format => $scsi_fmt,
 911     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 912 };
 913 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 914
 915 my $sata_fmt = {
 916     %drivedesc_base,
 917 };
 918 my $satadesc = {
 919     optional => 1,
 920     type => 'string', format => $sata_fmt,
 921     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 922 };
 923 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 924
 925 my $virtio_fmt = {
 926     %drivedesc_base,
 927     %iothread_fmt,
 928 };
 929 my $virtiodesc = {
 930     optional => 1,
 931     type => 'string', format => $virtio_fmt,
 932     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 933 };
 934 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 935
 936 my $alldrive_fmt = {
 937     %drivedesc_base,
 938     %iothread_fmt,
 939     %model_fmt,
 940     %queues_fmt,
 941     %scsiblock_fmt,
 942 };
 943
 944 my $efidisk_fmt = {
 945     volume => { alias => 'file' },
 946     file => {
 947         type => 'string',
 948         format => 'pve-volume-id-or-qm-path',
 949         default_key => 1,
 950         format_description => 'volume',
 951         description => "The drive's backing volume.",
 952     },
 953     format => get_standard_option('pve-qm-image-format'),
 954     size => {
 955         type => 'string',
 956         format => 'disk-size',
 957         format_description => 'DiskSize',
 958         description => "Disk size. This is purely informational and has no effect.",
 959         optional => 1,
 960     },
 961 };
 962
 963 my $efidisk_desc = {
 964     optional => 1,
 965     type => 'string', format => $efidisk_fmt,
 966     description => "Configure a Disk for storing EFI vars",
 967 };
 968
 969 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 970
 971 my $usb_fmt = {
 972     host => {
 973         default_key => 1,
 974         type => 'string', format => 'pve-qm-usb-device',
 975         format_description => 'HOSTUSBDEVICE|spice',
 976         description => <<EODESCR,
 977 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 978
 979  'bus-port(.port)*' (decimal numbers) or
 980  'vendor_id:product_id' (hexadeciaml numbers) or
 981  'spice'
 982
 983 You can use the 'lsusb -t' command to list existing usb devices.
 984
 985 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 986
 987 The value 'spice' can be used to add a usb redirection devices for spice.
 988 EODESCR
 989     },
 990     usb3 => {
 991         optional => 1,
 992         type => 'boolean',
 993         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 994         default => 0,
 995     },
 996 };
 997
 998 my $usbdesc = {
 999     optional => 1,
1000     type => 'string', format => $usb_fmt,
1001     description => "Configure an USB device (n is 0 to 4).",
1002 };
1003 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1004
1005 # NOTE: the match-groups of this regex are used in parse_hostpci
1006 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1007 my $hostpci_fmt = {
1008     host => {
1009         default_key => 1,
1010         type => 'string',
1011         pattern => qr/$PCIRE(;$PCIRE)*/,
1012         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1013         description => <<EODESCR,
1014 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1015 of PCI virtual functions of the host. HOSTPCIID syntax is:
1016
1017 'bus:dev.func' (hexadecimal numbers)
1018
1019 You can us the 'lspci' command to list existing PCI devices.
1020 EODESCR
1021     },
1022     rombar => {
1023         type => 'boolean',
1024         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1025         optional => 1,
1026         default => 1,
1027     },
1028     romfile => {
1029         type => 'string',
1030         pattern => '[^,;]+',
1031         format_description => 'string',
1032         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1033         optional => 1,
1034     },
1035     pcie => {
1036         type => 'boolean',
1037         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1038         optional => 1,
1039         default => 0,
1040     },
1041     'x-vga' => {
1042         type => 'boolean',
1043         description =>  "Enable vfio-vga device support.",
1044         optional => 1,
1045         default => 0,
1046     },
1047 };
1048 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1049
1050 my $hostpcidesc = {
1051         optional => 1,
1052         type => 'string', format => 'pve-qm-hostpci',
1053         description => "Map host PCI devices into guest.",
1054         verbose_description =>  <<EODESCR,
1055 Map host PCI devices into guest.
1056
1057 NOTE: This option allows direct access to host hardware. So it is no longer
1058 possible to migrate such machines - use with special care.
1059
1060 CAUTION: Experimental! User reported problems with this option.
1061 EODESCR
1062 };
1063 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1064
1065 my $serialdesc = {
1066         optional => 1,
1067         type => 'string',
1068         pattern => '(/dev/.+|socket)',
1069         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1070         verbose_description =>  <<EODESCR,
1071 Create a serial device inside the VM (n is 0 to 3), and pass through a
1072 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1073 host side (use 'qm terminal' to open a terminal connection).
1074
1075 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1076
1077 CAUTION: Experimental! User reported problems with this option.
1078 EODESCR
1079 };
1080
1081 my $paralleldesc= {
1082         optional => 1,
1083         type => 'string',
1084         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1085         description =>  "Map host parallel devices (n is 0 to 2).",
1086         verbose_description =>  <<EODESCR,
1087 Map host parallel devices (n is 0 to 2).
1088
1089 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1090
1091 CAUTION: Experimental! User reported problems with this option.
1092 EODESCR
1093 };
1094
1095 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1096     $confdesc->{"parallel$i"} = $paralleldesc;
1097 }
1098
1099 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1100     $confdesc->{"serial$i"} = $serialdesc;
1101 }
1102
1103 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1104     $confdesc->{"hostpci$i"} = $hostpcidesc;
1105 }
1106
1107 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1108     $drivename_hash->{"ide$i"} = 1;
1109     $confdesc->{"ide$i"} = $idedesc;
1110 }
1111
1112 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1113     $drivename_hash->{"sata$i"} = 1;
1114     $confdesc->{"sata$i"} = $satadesc;
1115 }
1116
1117 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1118     $drivename_hash->{"scsi$i"} = 1;
1119     $confdesc->{"scsi$i"} = $scsidesc ;
1120 }
1121
1122 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1123     $drivename_hash->{"virtio$i"} = 1;
1124     $confdesc->{"virtio$i"} = $virtiodesc;
1125 }
1126
1127 $drivename_hash->{efidisk0} = 1;
1128 $confdesc->{efidisk0} = $efidisk_desc;
1129
1130 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1131     $confdesc->{"usb$i"} = $usbdesc;
1132 }
1133
1134 my $unuseddesc = {
1135     optional => 1,
1136     type => 'string', format => 'pve-volume-id',
1137     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1138 };
1139
1140 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1141     $confdesc->{"unused$i"} = $unuseddesc;
1142 }
1143
1144 my $kvm_api_version = 0;
1145
1146 sub kvm_version {
1147
1148     return $kvm_api_version if $kvm_api_version;
1149
1150     my $fh = IO::File->new("</dev/kvm") ||
1151         return 0;
1152
1153     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1154         $kvm_api_version = $v;
1155     }
1156
1157     $fh->close();
1158
1159     return  $kvm_api_version;
1160 }
1161
1162 my $kvm_user_version;
1163
1164 sub kvm_user_version {
1165
1166     return $kvm_user_version if $kvm_user_version;
1167
1168     $kvm_user_version = 'unknown';
1169
1170     my $code = sub {
1171         my $line = shift;
1172         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1173             $kvm_user_version = $2;
1174         }
1175     };
1176
1177     eval { run_command("kvm -version", outfunc => $code); };
1178     warn $@ if $@;
1179
1180     return $kvm_user_version;
1181
1182 }
1183
1184 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1185
1186 sub valid_drive_names {
1187     # order is important - used to autoselect boot disk
1188     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1189             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1190             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1191             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1192             'efidisk0');
1193 }
1194
1195 sub is_valid_drivename {
1196     my $dev = shift;
1197
1198     return defined($drivename_hash->{$dev});
1199 }
1200
1201 sub option_exists {
1202     my $key = shift;
1203     return defined($confdesc->{$key});
1204 }
1205
1206 sub nic_models {
1207     return $nic_model_list;
1208 }
1209
1210 sub os_list_description {
1211
1212     return {
1213         other => 'Other',
1214         wxp => 'Windows XP',
1215         w2k => 'Windows 2000',
1216         w2k3 =>, 'Windows 2003',
1217         w2k8 => 'Windows 2008',
1218         wvista => 'Windows Vista',
1219         win7 => 'Windows 7',
1220         win8 => 'Windows 8/2012',
1221         win10 => 'Windows 10/2016',
1222         l24 => 'Linux 2.4',
1223         l26 => 'Linux 2.6',
1224     };
1225 }
1226
1227 my $cdrom_path;
1228
1229 sub get_cdrom_path {
1230
1231     return  $cdrom_path if $cdrom_path;
1232
1233     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1234     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1235     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1236 }
1237
1238 sub get_iso_path {
1239     my ($storecfg, $vmid, $cdrom) = @_;
1240
1241     if ($cdrom eq 'cdrom') {
1242         return get_cdrom_path();
1243     } elsif ($cdrom eq 'none') {
1244         return '';
1245     } elsif ($cdrom =~ m|^/|) {
1246         return $cdrom;
1247     } else {
1248         return PVE::Storage::path($storecfg, $cdrom);
1249     }
1250 }
1251
1252 # try to convert old style file names to volume IDs
1253 sub filename_to_volume_id {
1254     my ($vmid, $file, $media) = @_;
1255
1256     if (!($file eq 'none' || $file eq 'cdrom' ||
1257           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1258
1259         return undef if $file =~ m|/|;
1260
1261         if ($media && $media eq 'cdrom') {
1262             $file = "local:iso/$file";
1263         } else {
1264             $file = "local:$vmid/$file";
1265         }
1266     }
1267
1268     return $file;
1269 }
1270
1271 sub verify_media_type {
1272     my ($opt, $vtype, $media) = @_;
1273
1274     return if !$media;
1275
1276     my $etype;
1277     if ($media eq 'disk') {
1278         $etype = 'images';
1279     } elsif ($media eq 'cdrom') {
1280         $etype = 'iso';
1281     } else {
1282         die "internal error";
1283     }
1284
1285     return if ($vtype eq $etype);
1286
1287     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1288 }
1289
1290 sub cleanup_drive_path {
1291     my ($opt, $storecfg, $drive) = @_;
1292
1293     # try to convert filesystem paths to volume IDs
1294
1295     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1296         ($drive->{file} !~ m|^/dev/.+|) &&
1297         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1298         ($drive->{file} !~ m/^\d+$/)) {
1299         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1300         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1301         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1302         verify_media_type($opt, $vtype, $drive->{media});
1303         $drive->{file} = $volid;
1304     }
1305
1306     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1307 }
1308
1309 sub parse_hotplug_features {
1310     my ($data) = @_;
1311
1312     my $res = {};
1313
1314     return $res if $data eq '0';
1315
1316     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1317
1318     foreach my $feature (PVE::Tools::split_list($data)) {
1319         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1320             $res->{$1} = 1;
1321         } else {
1322             die "invalid hotplug feature '$feature'\n";
1323         }
1324     }
1325     return $res;
1326 }
1327
1328 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1329 sub pve_verify_hotplug_features {
1330     my ($value, $noerr) = @_;
1331
1332     return $value if parse_hotplug_features($value);
1333
1334     return undef if $noerr;
1335
1336     die "unable to parse hotplug option\n";
1337 }
1338
1339 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1340 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1341 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1342 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1343 #        [,iothread=on][,serial=serial][,model=model]
1344
1345 sub parse_drive {
1346     my ($key, $data) = @_;
1347
1348     my ($interface, $index);
1349
1350     if ($key =~ m/^([^\d]+)(\d+)$/) {
1351         $interface = $1;
1352         $index = $2;
1353     } else {
1354         return undef;
1355     }
1356
1357     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1358                                      : $confdesc->{$key}->{format};
1359     if (!$desc) {
1360         warn "invalid drive key: $key\n";
1361         return undef;
1362     }
1363     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1364     return undef if !$res;
1365     $res->{interface} = $interface;
1366     $res->{index} = $index;
1367
1368     my $error = 0;
1369     foreach my $opt (qw(bps bps_rd bps_wr)) {
1370         if (my $bps = defined(delete $res->{$opt})) {
1371             if (defined($res->{"m$opt"})) {
1372                 warn "both $opt and m$opt specified\n";
1373                 ++$error;
1374                 next;
1375             }
1376             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1377         }
1378     }
1379
1380     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1381     for my $requirement (
1382         [mbps_max => 'mbps'],
1383         [mbps_rd_max => 'mbps_rd'],
1384         [mbps_wr_max => 'mbps_wr'],
1385         [miops_max => 'miops'],
1386         [miops_rd_max => 'miops_rd'],
1387         [miops_wr_max => 'miops_wr'],
1388         [bps_max_length => 'mbps_max'],
1389         [bps_rd_max_length => 'mbps_rd_max'],
1390         [bps_wr_max_length => 'mbps_wr_max'],
1391         [iops_max_length => 'iops_max'],
1392         [iops_rd_max_length => 'iops_rd_max'],
1393         [iops_wr_max_length => 'iops_wr_max']) {
1394         my ($option, $requires) = @$requirement;
1395         if ($res->{$option} && !$res->{$requires}) {
1396             warn "$option requires $requires\n";
1397             ++$error;
1398         }
1399     }
1400
1401     return undef if $error;
1402
1403     return undef if $res->{mbps_rd} && $res->{mbps};
1404     return undef if $res->{mbps_wr} && $res->{mbps};
1405     return undef if $res->{iops_rd} && $res->{iops};
1406     return undef if $res->{iops_wr} && $res->{iops};
1407
1408     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1409         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1410         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1411         return undef if $res->{interface} eq 'virtio';
1412     }
1413
1414     if (my $size = $res->{size}) {
1415         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1416     }
1417
1418     return $res;
1419 }
1420
1421 sub print_drive {
1422     my ($vmid, $drive) = @_;
1423     my $data = { %$drive };
1424     delete $data->{$_} for qw(index interface);
1425     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1426 }
1427
1428 sub scsi_inquiry {
1429     my($fh, $noerr) = @_;
1430
1431     my $SG_IO = 0x2285;
1432     my $SG_GET_VERSION_NUM = 0x2282;
1433
1434     my $versionbuf = "\x00" x 8;
1435     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1436     if (!$ret) {
1437         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1438         return undef;
1439     }
1440     my $version = unpack("I", $versionbuf);
1441     if ($version < 30000) {
1442         die "scsi generic interface too old\n"  if !$noerr;
1443         return undef;
1444     }
1445
1446     my $buf = "\x00" x 36;
1447     my $sensebuf = "\x00" x 8;
1448     my $cmd = pack("C x3 C x1", 0x12, 36);
1449
1450     # see /usr/include/scsi/sg.h
1451     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1452
1453     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1454                       length($sensebuf), 0, length($buf), $buf,
1455                       $cmd, $sensebuf, 6000);
1456
1457     $ret = ioctl($fh, $SG_IO, $packet);
1458     if (!$ret) {
1459         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1460         return undef;
1461     }
1462
1463     my @res = unpack($sg_io_hdr_t, $packet);
1464     if ($res[17] || $res[18]) {
1465         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1466         return undef;
1467     }
1468
1469     my $res = {};
1470     (my $byte0, my $byte1, $res->{vendor},
1471      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1472
1473     $res->{removable} = $byte1 & 128 ? 1 : 0;
1474     $res->{type} = $byte0 & 31;
1475
1476     return $res;
1477 }
1478
1479 sub path_is_scsi {
1480     my ($path) = @_;
1481
1482     my $fh = IO::File->new("+<$path") || return undef;
1483     my $res = scsi_inquiry($fh, 1);
1484     close($fh);
1485
1486     return $res;
1487 }
1488
1489 sub machine_type_is_q35 {
1490     my ($conf) = @_;
1491
1492     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1493 }
1494
1495 sub print_tabletdevice_full {
1496     my ($conf) = @_;
1497
1498     my $q35 = machine_type_is_q35($conf);
1499
1500     # we use uhci for old VMs because tablet driver was buggy in older qemu
1501     my $usbbus = $q35 ? "ehci" : "uhci";
1502
1503     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1504 }
1505
1506 sub print_drivedevice_full {
1507     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1508
1509     my $device = '';
1510     my $maxdev = 0;
1511
1512     if ($drive->{interface} eq 'virtio') {
1513         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1514         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1515         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1516     } elsif ($drive->{interface} eq 'scsi') {
1517
1518         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1519         my $unit = $drive->{index} % $maxdev;
1520         my $devicetype = 'hd';
1521         my $path = '';
1522         if (drive_is_cdrom($drive)) {
1523             $devicetype = 'cd';
1524         } else {
1525             if ($drive->{file} =~ m|^/|) {
1526                 $path = $drive->{file};
1527                 if (my $info = path_is_scsi($path)) {
1528                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1529                         $devicetype = 'block';
1530                     } elsif ($info->{type} == 1) { # tape
1531                         $devicetype = 'generic';
1532                     }
1533                 }
1534             } else {
1535                  $path = PVE::Storage::path($storecfg, $drive->{file});
1536             }
1537
1538             if($path =~ m/^iscsi\:\/\//){
1539                 $devicetype = 'generic';
1540             }
1541         }
1542
1543         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1544             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1545         } else {
1546             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1547         }
1548
1549     } elsif ($drive->{interface} eq 'ide'){
1550         $maxdev = 2;
1551         my $controller = int($drive->{index} / $maxdev);
1552         my $unit = $drive->{index} % $maxdev;
1553         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1554
1555         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1556         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1557             $model = URI::Escape::uri_unescape($model);
1558             $device .= ",model=$model";
1559         }
1560     } elsif ($drive->{interface} eq 'sata'){
1561         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1562         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1563         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1564     } elsif ($drive->{interface} eq 'usb') {
1565         die "implement me";
1566         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1567     } else {
1568         die "unsupported interface type";
1569     }
1570
1571     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1572
1573     return $device;
1574 }
1575
1576 sub get_initiator_name {
1577     my $initiator;
1578
1579     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1580     while (defined(my $line = <$fh>)) {
1581         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1582         $initiator = $1;
1583         last;
1584     }
1585     $fh->close();
1586
1587     return $initiator;
1588 }
1589
1590 sub print_drive_full {
1591     my ($storecfg, $vmid, $drive) = @_;
1592
1593     my $path;
1594     my $volid = $drive->{file};
1595     my $format;
1596
1597     if (drive_is_cdrom($drive)) {
1598         $path = get_iso_path($storecfg, $vmid, $volid);
1599     } else {
1600         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1601         if ($storeid) {
1602             $path = PVE::Storage::path($storecfg, $volid);
1603             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1604             $format = qemu_img_format($scfg, $volname);
1605         } else {
1606             $path = $volid;
1607             $format = "raw";
1608         }
1609    }
1610
1611     my $opts = '';
1612     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard);
1613     foreach my $o (@qemu_drive_options) {
1614         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1615     }
1616     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1617         my ($dir, $qmpname) = @$type;
1618         if (my $v = $drive->{"mbps$dir"}) {
1619             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1620         }
1621         if (my $v = $drive->{"mbps${dir}_max"}) {
1622             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1623         }
1624         if (my $v = $drive->{"bps${dir}_max_length"}) {
1625             $opts .= ",throttling.bps$qmpname-max-length=$v";
1626         }
1627         if (my $v = $drive->{"iops${dir}"}) {
1628             $opts .= ",throttling.iops$qmpname=$v";
1629         }
1630         if (my $v = $drive->{"iops${dir}_max"}) {
1631             $opts .= ",throttling.iops$qmpname-max=$v";
1632         }
1633         if (my $v = $drive->{"iops${dir}_max_length"}) {
1634             $opts .= ",throttling.iops$qmpname-max-length=$v";
1635         }
1636     }
1637
1638     if (my $serial = $drive->{serial}) {
1639         $serial = URI::Escape::uri_unescape($serial);
1640         $opts .= ",serial=$serial";
1641     }
1642
1643     $opts .= ",format=$format" if $format && !$drive->{format};
1644
1645     my $cache_direct = 0;
1646
1647     if (my $cache = $drive->{cache}) {
1648         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1649     } elsif (!drive_is_cdrom($drive)) {
1650         $opts .= ",cache=none";
1651         $cache_direct = 1;
1652     }
1653
1654     # aio native works only with O_DIRECT
1655     if (!$drive->{aio}) {
1656         if($cache_direct) {
1657             $opts .= ",aio=native";
1658         } else {
1659             $opts .= ",aio=threads";
1660         }
1661     }
1662
1663     if (!drive_is_cdrom($drive)) {
1664         my $detectzeroes;
1665         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1666             $detectzeroes = 'off';
1667         } elsif ($drive->{discard}) {
1668             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1669         } else {
1670             # This used to be our default with discard not being specified:
1671             $detectzeroes = 'on';
1672         }
1673         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1674     }
1675
1676     my $pathinfo = $path ? "file=$path," : '';
1677
1678     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1679 }
1680
1681 sub print_netdevice_full {
1682     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1683
1684     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1685
1686     my $device = $net->{model};
1687     if ($net->{model} eq 'virtio') {
1688          $device = 'virtio-net-pci';
1689      };
1690
1691     my $pciaddr = print_pci_addr("$netid", $bridges);
1692     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1693     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1694         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1695         my $vectors = $net->{queues} * 2 + 2;
1696         $tmpstr .= ",vectors=$vectors,mq=on";
1697     }
1698     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1699
1700     if ($use_old_bios_files) {
1701         my $romfile;
1702         if ($device eq 'virtio-net-pci') {
1703             $romfile = 'pxe-virtio.rom';
1704         } elsif ($device eq 'e1000') {
1705             $romfile = 'pxe-e1000.rom';
1706         } elsif ($device eq 'ne2k') {
1707             $romfile = 'pxe-ne2k_pci.rom';
1708         } elsif ($device eq 'pcnet') {
1709             $romfile = 'pxe-pcnet.rom';
1710         } elsif ($device eq 'rtl8139') {
1711             $romfile = 'pxe-rtl8139.rom';
1712         }
1713         $tmpstr .= ",romfile=$romfile" if $romfile;
1714     }
1715
1716     return $tmpstr;
1717 }
1718
1719 sub print_netdev_full {
1720     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1721
1722     my $i = '';
1723     if ($netid =~ m/^net(\d+)$/) {
1724         $i = int($1);
1725     }
1726
1727     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1728
1729     my $ifname = "tap${vmid}i$i";
1730
1731     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1732     die "interface name '$ifname' is too long (max 15 character)\n"
1733         if length($ifname) >= 16;
1734
1735     my $vhostparam = '';
1736     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1737
1738     my $vmname = $conf->{name} || "vm$vmid";
1739
1740     my $netdev = "";
1741     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1742
1743     if ($net->{bridge}) {
1744         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1745     } else {
1746         $netdev = "type=user,id=$netid,hostname=$vmname";
1747     }
1748
1749     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1750
1751     return $netdev;
1752 }
1753
1754
1755 sub print_cpu_device {
1756     my ($conf, $id) = @_;
1757
1758     my $kvm = $conf->{kvm} // 1;
1759     my $cpu = $kvm ? "kvm64" : "qemu64";
1760     if (my $cputype = $conf->{cpu}) {
1761         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1762             or die "Cannot parse cpu description: $cputype\n";
1763         $cpu = $cpuconf->{cputype};
1764     }
1765
1766     my $cores = $conf->{cores} || 1;
1767
1768     my $current_core = ($id - 1) % $cores;
1769     my $current_socket = int(($id - 1 - $current_core)/$cores);
1770
1771     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1772 }
1773
1774 sub drive_is_cdrom {
1775     my ($drive) = @_;
1776
1777     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1778
1779 }
1780
1781 sub parse_number_sets {
1782     my ($set) = @_;
1783     my $res = [];
1784     foreach my $part (split(/;/, $set)) {
1785         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1786             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1787             push @$res, [ $1, $2 ];
1788         } else {
1789             die "invalid range: $part\n";
1790         }
1791     }
1792     return $res;
1793 }
1794
1795 sub parse_numa {
1796     my ($data) = @_;
1797
1798     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1799     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1800     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1801     return $res;
1802 }
1803
1804 sub parse_hostpci {
1805     my ($value) = @_;
1806
1807     return undef if !$value;
1808
1809     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1810
1811     my @idlist = split(/;/, $res->{host});
1812     delete $res->{host};
1813     foreach my $id (@idlist) {
1814         if ($id =~ /^$PCIRE$/) {
1815             if (defined($2)) {
1816                 push @{$res->{pciid}}, { id => $1, function => $2 };
1817             } else {
1818                 my $pcidevices = lspci($1);
1819                 $res->{pciid} = $pcidevices->{$1};
1820             }
1821         } else {
1822             # should have been caught by parse_property_string already
1823             die "failed to parse PCI id: $id\n";
1824         }
1825     }
1826     return $res;
1827 }
1828
1829 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1830 sub parse_net {
1831     my ($data) = @_;
1832
1833     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1834     if ($@) {
1835         warn $@;
1836         return undef;
1837     }
1838     if (!defined($res->{macaddr})) {
1839         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1840         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1841     }
1842     return $res;
1843 }
1844
1845 sub print_net {
1846     my $net = shift;
1847
1848     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1849 }
1850
1851 sub add_random_macs {
1852     my ($settings) = @_;
1853
1854     foreach my $opt (keys %$settings) {
1855         next if $opt !~ m/^net(\d+)$/;
1856         my $net = parse_net($settings->{$opt});
1857         next if !$net;
1858         $settings->{$opt} = print_net($net);
1859     }
1860 }
1861
1862 sub vm_is_volid_owner {
1863     my ($storecfg, $vmid, $volid) = @_;
1864
1865     if ($volid !~  m|^/|) {
1866         my ($path, $owner);
1867         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1868         if ($owner && ($owner == $vmid)) {
1869             return 1;
1870         }
1871     }
1872
1873     return undef;
1874 }
1875
1876 sub split_flagged_list {
1877     my $text = shift || '';
1878     $text =~ s/[,;]/ /g;
1879     $text =~ s/^\s+//;
1880     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1881 }
1882
1883 sub join_flagged_list {
1884     my ($how, $lst) = @_;
1885     join $how, map { $lst->{$_} . $_ } keys %$lst;
1886 }
1887
1888 sub vmconfig_delete_pending_option {
1889     my ($conf, $key, $force) = @_;
1890
1891     delete $conf->{pending}->{$key};
1892     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1893     $pending_delete_hash->{$key} = $force ? '!' : '';
1894     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1895 }
1896
1897 sub vmconfig_undelete_pending_option {
1898     my ($conf, $key) = @_;
1899
1900     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1901     delete $pending_delete_hash->{$key};
1902
1903     if (%$pending_delete_hash) {
1904         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1905     } else {
1906         delete $conf->{pending}->{delete};
1907     }
1908 }
1909
1910 sub vmconfig_register_unused_drive {
1911     my ($storecfg, $vmid, $conf, $drive) = @_;
1912
1913     if (!drive_is_cdrom($drive)) {
1914         my $volid = $drive->{file};
1915         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1916             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1917         }
1918     }
1919 }
1920
1921 sub vmconfig_cleanup_pending {
1922     my ($conf) = @_;
1923
1924     # remove pending changes when nothing changed
1925     my $changes;
1926     foreach my $opt (keys %{$conf->{pending}}) {
1927         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1928             $changes = 1;
1929             delete $conf->{pending}->{$opt};
1930         }
1931     }
1932
1933     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1934     my $pending_delete_hash = {};
1935     while (my ($opt, $force) = each %$current_delete_hash) {
1936         if (defined($conf->{$opt})) {
1937             $pending_delete_hash->{$opt} = $force;
1938         } else {
1939             $changes = 1;
1940         }
1941     }
1942
1943     if (%$pending_delete_hash) {
1944         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1945     } else {
1946         delete $conf->{pending}->{delete};
1947     }
1948
1949     return $changes;
1950 }
1951
1952 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1953 my $smbios1_fmt = {
1954     uuid => {
1955         type => 'string',
1956         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1957         format_description => 'UUID',
1958         description => "Set SMBIOS1 UUID.",
1959         optional => 1,
1960     },
1961     version => {
1962         type => 'string',
1963         pattern => '\S+',
1964         format_description => 'string',
1965         description => "Set SMBIOS1 version.",
1966         optional => 1,
1967     },
1968     serial => {
1969         type => 'string',
1970         pattern => '\S+',
1971         format_description => 'string',
1972         description => "Set SMBIOS1 serial number.",
1973         optional => 1,
1974     },
1975     manufacturer => {
1976         type => 'string',
1977         pattern => '\S+',
1978         format_description => 'string',
1979         description => "Set SMBIOS1 manufacturer.",
1980         optional => 1,
1981     },
1982     product => {
1983         type => 'string',
1984         pattern => '\S+',
1985         format_description => 'string',
1986         description => "Set SMBIOS1 product ID.",
1987         optional => 1,
1988     },
1989     sku => {
1990         type => 'string',
1991         pattern => '\S+',
1992         format_description => 'string',
1993         description => "Set SMBIOS1 SKU string.",
1994         optional => 1,
1995     },
1996     family => {
1997         type => 'string',
1998         pattern => '\S+',
1999         format_description => 'string',
2000         description => "Set SMBIOS1 family string.",
2001         optional => 1,
2002     },
2003 };
2004
2005 sub parse_smbios1 {
2006     my ($data) = @_;
2007
2008     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2009     warn $@ if $@;
2010     return $res;
2011 }
2012
2013 sub print_smbios1 {
2014     my ($smbios1) = @_;
2015     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2016 }
2017
2018 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2019
2020 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2021 sub verify_bootdisk {
2022     my ($value, $noerr) = @_;
2023
2024     return $value if is_valid_drivename($value);
2025
2026     return undef if $noerr;
2027
2028     die "invalid boot disk '$value'\n";
2029 }
2030
2031 sub parse_watchdog {
2032     my ($value) = @_;
2033
2034     return undef if !$value;
2035
2036     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2037     warn $@ if $@;
2038     return $res;
2039 }
2040
2041 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2042 sub verify_usb_device {
2043     my ($value, $noerr) = @_;
2044
2045     return $value if parse_usb_device($value);
2046
2047     return undef if $noerr;
2048
2049     die "unable to parse usb device\n";
2050 }
2051
2052 # add JSON properties for create and set function
2053 sub json_config_properties {
2054     my $prop = shift;
2055
2056     foreach my $opt (keys %$confdesc) {
2057         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2058         $prop->{$opt} = $confdesc->{$opt};
2059     }
2060
2061     return $prop;
2062 }
2063
2064 sub check_type {
2065     my ($key, $value) = @_;
2066
2067     die "unknown setting '$key'\n" if !$confdesc->{$key};
2068
2069     my $type = $confdesc->{$key}->{type};
2070
2071     if (!defined($value)) {
2072         die "got undefined value\n";
2073     }
2074
2075     if ($value =~ m/[\n\r]/) {
2076         die "property contains a line feed\n";
2077     }
2078
2079     if ($type eq 'boolean') {
2080         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2081         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2082         die "type check ('boolean') failed - got '$value'\n";
2083     } elsif ($type eq 'integer') {
2084         return int($1) if $value =~ m/^(\d+)$/;
2085         die "type check ('integer') failed - got '$value'\n";
2086     } elsif ($type eq 'number') {
2087         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2088         die "type check ('number') failed - got '$value'\n";
2089     } elsif ($type eq 'string') {
2090         if (my $fmt = $confdesc->{$key}->{format}) {
2091             PVE::JSONSchema::check_format($fmt, $value);
2092             return $value;
2093         }
2094         $value =~ s/^\"(.*)\"$/$1/;
2095         return $value;
2096     } else {
2097         die "internal error"
2098     }
2099 }
2100
2101 sub check_iommu_support{
2102     #fixme : need to check IOMMU support
2103     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2104
2105     my $iommu=1;
2106     return $iommu;
2107
2108 }
2109
2110 sub touch_config {
2111     my ($vmid) = @_;
2112
2113     my $conf = PVE::QemuConfig->config_file($vmid);
2114     utime undef, undef, $conf;
2115 }
2116
2117 sub destroy_vm {
2118     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2119
2120     my $conffile = PVE::QemuConfig->config_file($vmid);
2121
2122     my $conf = PVE::QemuConfig->load_config($vmid);
2123
2124     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2125
2126     if ($conf->{template}) {
2127         # check if any base image is still used by a linked clone
2128         foreach_drive($conf, sub {
2129                 my ($ds, $drive) = @_;
2130
2131                 return if drive_is_cdrom($drive);
2132
2133                 my $volid = $drive->{file};
2134
2135                 return if !$volid || $volid =~ m|^/|;
2136
2137                 die "base volume '$volid' is still in use by linked cloned\n"
2138                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2139
2140         });
2141     }
2142
2143     # only remove disks owned by this VM
2144     foreach_drive($conf, sub {
2145         my ($ds, $drive) = @_;
2146
2147         return if drive_is_cdrom($drive);
2148
2149         my $volid = $drive->{file};
2150
2151         return if !$volid || $volid =~ m|^/|;
2152
2153         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2154         return if !$path || !$owner || ($owner != $vmid);
2155
2156         eval {
2157             PVE::Storage::vdisk_free($storecfg, $volid);
2158         };
2159         warn "Could not remove disk '$volid', check manually: $@" if $@;
2160
2161     });
2162
2163     if ($keep_empty_config) {
2164         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2165     } else {
2166         unlink $conffile;
2167     }
2168
2169     # also remove unused disk
2170     eval {
2171         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2172
2173         eval {
2174             PVE::Storage::foreach_volid($dl, sub {
2175                 my ($volid, $sid, $volname, $d) = @_;
2176                 PVE::Storage::vdisk_free($storecfg, $volid);
2177             });
2178         };
2179         warn $@ if $@;
2180
2181     };
2182     warn $@ if $@;
2183 }
2184
2185 sub parse_vm_config {
2186     my ($filename, $raw) = @_;
2187
2188     return undef if !defined($raw);
2189
2190     my $res = {
2191         digest => Digest::SHA::sha1_hex($raw),
2192         snapshots => {},
2193         pending => {},
2194     };
2195
2196     $filename =~ m|/qemu-server/(\d+)\.conf$|
2197         || die "got strange filename '$filename'";
2198
2199     my $vmid = $1;
2200
2201     my $conf = $res;
2202     my $descr;
2203     my $section = '';
2204
2205     my @lines = split(/\n/, $raw);
2206     foreach my $line (@lines) {
2207         next if $line =~ m/^\s*$/;
2208
2209         if ($line =~ m/^\[PENDING\]\s*$/i) {
2210             $section = 'pending';
2211             if (defined($descr)) {
2212                 $descr =~ s/\s+$//;
2213                 $conf->{description} = $descr;
2214             }
2215             $descr = undef;
2216             $conf = $res->{$section} = {};
2217             next;
2218
2219         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2220             $section = $1;
2221             if (defined($descr)) {
2222                 $descr =~ s/\s+$//;
2223                 $conf->{description} = $descr;
2224             }
2225             $descr = undef;
2226             $conf = $res->{snapshots}->{$section} = {};
2227             next;
2228         }
2229
2230         if ($line =~ m/^\#(.*)\s*$/) {
2231             $descr = '' if !defined($descr);
2232             $descr .= PVE::Tools::decode_text($1) . "\n";
2233             next;
2234         }
2235
2236         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2237             $descr = '' if !defined($descr);
2238             $descr .= PVE::Tools::decode_text($2);
2239         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2240             $conf->{snapstate} = $1;
2241         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2242             my $key = $1;
2243             my $value = $2;
2244             $conf->{$key} = $value;
2245         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2246             my $value = $1;
2247             if ($section eq 'pending') {
2248                 $conf->{delete} = $value; # we parse this later
2249             } else {
2250                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2251             }
2252         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2253             my $key = $1;
2254             my $value = $2;
2255             eval { $value = check_type($key, $value); };
2256             if ($@) {
2257                 warn "vm $vmid - unable to parse value of '$key' - $@";
2258             } else {
2259                 $key = 'ide2' if $key eq 'cdrom';
2260                 my $fmt = $confdesc->{$key}->{format};
2261                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2262                     my $v = parse_drive($key, $value);
2263                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2264                         $v->{file} = $volid;
2265                         $value = print_drive($vmid, $v);
2266                     } else {
2267                         warn "vm $vmid - unable to parse value of '$key'\n";
2268                         next;
2269                     }
2270                 }
2271
2272                 $conf->{$key} = $value;
2273             }
2274         }
2275     }
2276
2277     if (defined($descr)) {
2278         $descr =~ s/\s+$//;
2279         $conf->{description} = $descr;
2280     }
2281     delete $res->{snapstate}; # just to be sure
2282
2283     return $res;
2284 }
2285
2286 sub write_vm_config {
2287     my ($filename, $conf) = @_;
2288
2289     delete $conf->{snapstate}; # just to be sure
2290
2291     if ($conf->{cdrom}) {
2292         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2293         $conf->{ide2} = $conf->{cdrom};
2294         delete $conf->{cdrom};
2295     }
2296
2297     # we do not use 'smp' any longer
2298     if ($conf->{sockets}) {
2299         delete $conf->{smp};
2300     } elsif ($conf->{smp}) {
2301         $conf->{sockets} = $conf->{smp};
2302         delete $conf->{cores};
2303         delete $conf->{smp};
2304     }
2305
2306     my $used_volids = {};
2307
2308     my $cleanup_config = sub {
2309         my ($cref, $pending, $snapname) = @_;
2310
2311         foreach my $key (keys %$cref) {
2312             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2313                 $key eq 'snapstate' || $key eq 'pending';
2314             my $value = $cref->{$key};
2315             if ($key eq 'delete') {
2316                 die "propertry 'delete' is only allowed in [PENDING]\n"
2317                     if !$pending;
2318                 # fixme: check syntax?
2319                 next;
2320             }
2321             eval { $value = check_type($key, $value); };
2322             die "unable to parse value of '$key' - $@" if $@;
2323
2324             $cref->{$key} = $value;
2325
2326             if (!$snapname && is_valid_drivename($key)) {
2327                 my $drive = parse_drive($key, $value);
2328                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2329             }
2330         }
2331     };
2332
2333     &$cleanup_config($conf);
2334
2335     &$cleanup_config($conf->{pending}, 1);
2336
2337     foreach my $snapname (keys %{$conf->{snapshots}}) {
2338         die "internal error" if $snapname eq 'pending';
2339         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2340     }
2341
2342     # remove 'unusedX' settings if we re-add a volume
2343     foreach my $key (keys %$conf) {
2344         my $value = $conf->{$key};
2345         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2346             delete $conf->{$key};
2347         }
2348     }
2349
2350     my $generate_raw_config = sub {
2351         my ($conf, $pending) = @_;
2352
2353         my $raw = '';
2354
2355         # add description as comment to top of file
2356         if (defined(my $descr = $conf->{description})) {
2357             if ($descr) {
2358                 foreach my $cl (split(/\n/, $descr)) {
2359                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2360                 }
2361             } else {
2362                 $raw .= "#\n" if $pending;
2363             }
2364         }
2365
2366         foreach my $key (sort keys %$conf) {
2367             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2368             $raw .= "$key: $conf->{$key}\n";
2369         }
2370         return $raw;
2371     };
2372
2373     my $raw = &$generate_raw_config($conf);
2374
2375     if (scalar(keys %{$conf->{pending}})){
2376         $raw .= "\n[PENDING]\n";
2377         $raw .= &$generate_raw_config($conf->{pending}, 1);
2378     }
2379
2380     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2381         $raw .= "\n[$snapname]\n";
2382         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2383     }
2384
2385     return $raw;
2386 }
2387
2388 sub load_defaults {
2389
2390     my $res = {};
2391
2392     # we use static defaults from our JSON schema configuration
2393     foreach my $key (keys %$confdesc) {
2394         if (defined(my $default = $confdesc->{$key}->{default})) {
2395             $res->{$key} = $default;
2396         }
2397     }
2398
2399     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2400     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2401
2402     return $res;
2403 }
2404
2405 sub config_list {
2406     my $vmlist = PVE::Cluster::get_vmlist();
2407     my $res = {};
2408     return $res if !$vmlist || !$vmlist->{ids};
2409     my $ids = $vmlist->{ids};
2410
2411     foreach my $vmid (keys %$ids) {
2412         my $d = $ids->{$vmid};
2413         next if !$d->{node} || $d->{node} ne $nodename;
2414         next if !$d->{type} || $d->{type} ne 'qemu';
2415         $res->{$vmid}->{exists} = 1;
2416     }
2417     return $res;
2418 }
2419
2420 # test if VM uses local resources (to prevent migration)
2421 sub check_local_resources {
2422     my ($conf, $noerr) = @_;
2423
2424     my $loc_res = 0;
2425
2426     $loc_res = 1 if $conf->{hostusb}; # old syntax
2427     $loc_res = 1 if $conf->{hostpci}; # old syntax
2428
2429     foreach my $k (keys %$conf) {
2430         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2431         # sockets are safe: they will recreated be on the target side post-migrate
2432         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2433         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2434     }
2435
2436     die "VM uses local resources\n" if $loc_res && !$noerr;
2437
2438     return $loc_res;
2439 }
2440
2441 # check if used storages are available on all nodes (use by migrate)
2442 sub check_storage_availability {
2443     my ($storecfg, $conf, $node) = @_;
2444
2445     foreach_drive($conf, sub {
2446         my ($ds, $drive) = @_;
2447
2448         my $volid = $drive->{file};
2449         return if !$volid;
2450
2451         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2452         return if !$sid;
2453
2454         # check if storage is available on both nodes
2455         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2456         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2457    });
2458 }
2459
2460 # list nodes where all VM images are available (used by has_feature API)
2461 sub shared_nodes {
2462     my ($conf, $storecfg) = @_;
2463
2464     my $nodelist = PVE::Cluster::get_nodelist();
2465     my $nodehash = { map { $_ => 1 } @$nodelist };
2466     my $nodename = PVE::INotify::nodename();
2467
2468     foreach_drive($conf, sub {
2469         my ($ds, $drive) = @_;
2470
2471         my $volid = $drive->{file};
2472         return if !$volid;
2473
2474         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2475         if ($storeid) {
2476             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2477             if ($scfg->{disable}) {
2478                 $nodehash = {};
2479             } elsif (my $avail = $scfg->{nodes}) {
2480                 foreach my $node (keys %$nodehash) {
2481                     delete $nodehash->{$node} if !$avail->{$node};
2482                 }
2483             } elsif (!$scfg->{shared}) {
2484                 foreach my $node (keys %$nodehash) {
2485                     delete $nodehash->{$node} if $node ne $nodename
2486                 }
2487             }
2488         }
2489     });
2490
2491     return $nodehash
2492 }
2493
2494 sub check_cmdline {
2495     my ($pidfile, $pid) = @_;
2496
2497     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2498     if (defined($fh)) {
2499         my $line = <$fh>;
2500         $fh->close;
2501         return undef if !$line;
2502         my @param = split(/\0/, $line);
2503
2504         my $cmd = $param[0];
2505         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2506
2507         for (my $i = 0; $i < scalar (@param); $i++) {
2508             my $p = $param[$i];
2509             next if !$p;
2510             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2511                 my $p = $param[$i+1];
2512                 return 1 if $p && ($p eq $pidfile);
2513                 return undef;
2514             }
2515         }
2516     }
2517     return undef;
2518 }
2519
2520 sub check_running {
2521     my ($vmid, $nocheck, $node) = @_;
2522
2523     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2524
2525     die "unable to find configuration file for VM $vmid - no such machine\n"
2526         if !$nocheck && ! -f $filename;
2527
2528     my $pidfile = pidfile_name($vmid);
2529
2530     if (my $fd = IO::File->new("<$pidfile")) {
2531         my $st = stat($fd);
2532         my $line = <$fd>;
2533         close($fd);
2534
2535         my $mtime = $st->mtime;
2536         if ($mtime > time()) {
2537             warn "file '$filename' modified in future\n";
2538         }
2539
2540         if ($line =~ m/^(\d+)$/) {
2541             my $pid = $1;
2542             if (check_cmdline($pidfile, $pid)) {
2543                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2544                     return $pid;
2545                 }
2546             }
2547         }
2548     }
2549
2550     return undef;
2551 }
2552
2553 sub vzlist {
2554
2555     my $vzlist = config_list();
2556
2557     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2558
2559     while (defined(my $de = $fd->read)) {
2560         next if $de !~ m/^(\d+)\.pid$/;
2561         my $vmid = $1;
2562         next if !defined($vzlist->{$vmid});
2563         if (my $pid = check_running($vmid)) {
2564             $vzlist->{$vmid}->{pid} = $pid;
2565         }
2566     }
2567
2568     return $vzlist;
2569 }
2570
2571 sub disksize {
2572     my ($storecfg, $conf) = @_;
2573
2574     my $bootdisk = $conf->{bootdisk};
2575     return undef if !$bootdisk;
2576     return undef if !is_valid_drivename($bootdisk);
2577
2578     return undef if !$conf->{$bootdisk};
2579
2580     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2581     return undef if !defined($drive);
2582
2583     return undef if drive_is_cdrom($drive);
2584
2585     my $volid = $drive->{file};
2586     return undef if !$volid;
2587
2588     return $drive->{size};
2589 }
2590
2591 my $last_proc_pid_stat;
2592
2593 # get VM status information
2594 # This must be fast and should not block ($full == false)
2595 # We only query KVM using QMP if $full == true (this can be slow)
2596 sub vmstatus {
2597     my ($opt_vmid, $full) = @_;
2598
2599     my $res = {};
2600
2601     my $storecfg = PVE::Storage::config();
2602
2603     my $list = vzlist();
2604     my $defaults = load_defaults();
2605
2606     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2607
2608     my $cpucount = $cpuinfo->{cpus} || 1;
2609
2610     foreach my $vmid (keys %$list) {
2611         next if $opt_vmid && ($vmid ne $opt_vmid);
2612
2613         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2614         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2615
2616         my $d = {};
2617         $d->{pid} = $list->{$vmid}->{pid};
2618
2619         # fixme: better status?
2620         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2621
2622         my $size = disksize($storecfg, $conf);
2623         if (defined($size)) {
2624             $d->{disk} = 0; # no info available
2625             $d->{maxdisk} = $size;
2626         } else {
2627             $d->{disk} = 0;
2628             $d->{maxdisk} = 0;
2629         }
2630
2631         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2632             * ($conf->{cores} || $defaults->{cores});
2633         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2634         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2635
2636         $d->{name} = $conf->{name} || "VM $vmid";
2637         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2638             : $defaults->{memory}*(1024*1024);
2639
2640         if ($conf->{balloon}) {
2641             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2642             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2643                 : $defaults->{shares};
2644         }
2645
2646         $d->{uptime} = 0;
2647         $d->{cpu} = 0;
2648         $d->{mem} = 0;
2649
2650         $d->{netout} = 0;
2651         $d->{netin} = 0;
2652
2653         $d->{diskread} = 0;
2654         $d->{diskwrite} = 0;
2655
2656         $d->{template} = PVE::QemuConfig->is_template($conf);
2657
2658         $res->{$vmid} = $d;
2659     }
2660
2661     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2662     foreach my $dev (keys %$netdev) {
2663         next if $dev !~ m/^tap([1-9]\d*)i/;
2664         my $vmid = $1;
2665         my $d = $res->{$vmid};
2666         next if !$d;
2667
2668         $d->{netout} += $netdev->{$dev}->{receive};
2669         $d->{netin} += $netdev->{$dev}->{transmit};
2670
2671         if ($full) {
2672             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2673             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2674         }
2675
2676     }
2677
2678     my $ctime = gettimeofday;
2679
2680     foreach my $vmid (keys %$list) {
2681
2682         my $d = $res->{$vmid};
2683         my $pid = $d->{pid};
2684         next if !$pid;
2685
2686         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2687         next if !$pstat; # not running
2688
2689         my $used = $pstat->{utime} + $pstat->{stime};
2690
2691         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2692
2693         if ($pstat->{vsize}) {
2694             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2695         }
2696
2697         my $old = $last_proc_pid_stat->{$pid};
2698         if (!$old) {
2699             $last_proc_pid_stat->{$pid} = {
2700                 time => $ctime,
2701                 used => $used,
2702                 cpu => 0,
2703             };
2704             next;
2705         }
2706
2707         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2708
2709         if ($dtime > 1000) {
2710             my $dutime = $used -  $old->{used};
2711
2712             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2713             $last_proc_pid_stat->{$pid} = {
2714                 time => $ctime,
2715                 used => $used,
2716                 cpu => $d->{cpu},
2717             };
2718         } else {
2719             $d->{cpu} = $old->{cpu};
2720         }
2721     }
2722
2723     return $res if !$full;
2724
2725     my $qmpclient = PVE::QMPClient->new();
2726
2727     my $ballooncb = sub {
2728         my ($vmid, $resp) = @_;
2729
2730         my $info = $resp->{'return'};
2731         return if !$info->{max_mem};
2732
2733         my $d = $res->{$vmid};
2734
2735         # use memory assigned to VM
2736         $d->{maxmem} = $info->{max_mem};
2737         $d->{balloon} = $info->{actual};
2738
2739         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2740             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2741             $d->{freemem} = $info->{free_mem};
2742         }
2743
2744         $d->{ballooninfo} = $info;
2745     };
2746
2747     my $blockstatscb = sub {
2748         my ($vmid, $resp) = @_;
2749         my $data = $resp->{'return'} || [];
2750         my $totalrdbytes = 0;
2751         my $totalwrbytes = 0;
2752
2753         for my $blockstat (@$data) {
2754             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2755             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2756
2757             $blockstat->{device} =~ s/drive-//;
2758             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2759         }
2760         $res->{$vmid}->{diskread} = $totalrdbytes;
2761         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2762     };
2763
2764     my $statuscb = sub {
2765         my ($vmid, $resp) = @_;
2766
2767         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2768         # this fails if ballon driver is not loaded, so this must be
2769         # the last commnand (following command are aborted if this fails).
2770         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2771
2772         my $status = 'unknown';
2773         if (!defined($status = $resp->{'return'}->{status})) {
2774             warn "unable to get VM status\n";
2775             return;
2776         }
2777
2778         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2779     };
2780
2781     foreach my $vmid (keys %$list) {
2782         next if $opt_vmid && ($vmid ne $opt_vmid);
2783         next if !$res->{$vmid}->{pid}; # not running
2784         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2785     }
2786
2787     $qmpclient->queue_execute(undef, 2);
2788
2789     foreach my $vmid (keys %$list) {
2790         next if $opt_vmid && ($vmid ne $opt_vmid);
2791         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2792     }
2793
2794     return $res;
2795 }
2796
2797 sub foreach_drive {
2798     my ($conf, $func, @param) = @_;
2799
2800     foreach my $ds (valid_drive_names()) {
2801         next if !defined($conf->{$ds});
2802
2803         my $drive = parse_drive($ds, $conf->{$ds});
2804         next if !$drive;
2805
2806         &$func($ds, $drive, @param);
2807     }
2808 }
2809
2810 sub foreach_volid {
2811     my ($conf, $func, @param) = @_;
2812
2813     my $volhash = {};
2814
2815     my $test_volid = sub {
2816         my ($volid, $is_cdrom, $replicate, $snapname) = @_;
2817
2818         return if !$volid;
2819
2820         $volhash->{$volid}->{cdrom} //= 1;
2821         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2822
2823         $volhash->{$volid}->{replicate} //= 0;
2824         $volhash->{$volid}->{replicate} = 1 if $replicate;
2825
2826         $volhash->{$volid}->{referenced_in_config} //= 0;
2827         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
2828
2829         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
2830             if defined($snapname);
2831     };
2832
2833     foreach_drive($conf, sub {
2834         my ($ds, $drive) = @_;
2835         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, undef);
2836     });
2837
2838     foreach my $snapname (keys %{$conf->{snapshots}}) {
2839         my $snap = $conf->{snapshots}->{$snapname};
2840         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
2841         foreach_drive($snap, sub {
2842             my ($ds, $drive) = @_;
2843             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $snapname);
2844         });
2845     }
2846
2847     foreach my $volid (keys %$volhash) {
2848         &$func($volid, $volhash->{$volid}, @param);
2849     }
2850 }
2851
2852 sub vga_conf_has_spice {
2853     my ($vga) = @_;
2854
2855     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2856
2857     return $1 || 1;
2858 }
2859
2860 sub config_to_command {
2861     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2862
2863     my $cmd = [];
2864     my $globalFlags = [];
2865     my $machineFlags = [];
2866     my $rtcFlags = [];
2867     my $cpuFlags = [];
2868     my $devices = [];
2869     my $pciaddr = '';
2870     my $bridges = {};
2871     my $kvmver = kvm_user_version();
2872     my $vernum = 0; # unknown
2873     my $ostype = $conf->{ostype};
2874     my $winversion = windows_version($ostype);
2875     my $kvm = $conf->{kvm} // 1;
2876
2877     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
2878
2879     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2880         $vernum = $1*1000000+$2*1000;
2881     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2882         $vernum = $1*1000000+$2*1000+$3;
2883     }
2884
2885     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2886
2887     my $have_ovz = -f '/proc/vz/vestat';
2888
2889     my $q35 = machine_type_is_q35($conf);
2890     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2891     my $machine_type = $forcemachine || $conf->{machine};
2892     my $use_old_bios_files = undef;
2893     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2894
2895     my $cpuunits = defined($conf->{cpuunits}) ?
2896             $conf->{cpuunits} : $defaults->{cpuunits};
2897
2898     push @$cmd, '/usr/bin/kvm';
2899
2900     push @$cmd, '-id', $vmid;
2901
2902     my $use_virtio = 0;
2903
2904     my $qmpsocket = qmp_socket($vmid);
2905     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2906     push @$cmd, '-mon', "chardev=qmp,mode=control";
2907
2908
2909     push @$cmd, '-pidfile' , pidfile_name($vmid);
2910
2911     push @$cmd, '-daemonize';
2912
2913     if ($conf->{smbios1}) {
2914         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2915     }
2916
2917     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2918         die "uefi base image not found\n" if ! -f $OVMF_CODE;
2919
2920         my $path;
2921         my $format;
2922         if (my $efidisk = $conf->{efidisk0}) {
2923             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
2924             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2925             $format = $d->{format};
2926             if ($storeid) {
2927                 $path = PVE::Storage::path($storecfg, $d->{file});
2928                 if (!defined($format)) {
2929                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2930                     $format = qemu_img_format($scfg, $volname);
2931                 }
2932             } else {
2933                 $path = $d->{file};
2934                 die "efidisk format must be specified\n"
2935                     if !defined($format);
2936             }
2937         } else {
2938             warn "no efidisk configured! Using temporary efivars disk.\n";
2939             $path = "/tmp/$vmid-ovmf.fd";
2940             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
2941             $format = 'raw';
2942         }
2943
2944         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
2945         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
2946     }
2947
2948
2949     # add usb controllers
2950     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2951     push @$devices, @usbcontrollers if @usbcontrollers;
2952     my $vga = $conf->{vga};
2953
2954     my $qxlnum = vga_conf_has_spice($vga);
2955     $vga = 'qxl' if $qxlnum;
2956
2957     if (!$vga) {
2958         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
2959             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
2960         } else {
2961             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
2962         }
2963     }
2964
2965     # enable absolute mouse coordinates (needed by vnc)
2966     my $tablet;
2967     if (defined($conf->{tablet})) {
2968         $tablet = $conf->{tablet};
2969     } else {
2970         $tablet = $defaults->{tablet};
2971         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2972         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2973     }
2974
2975     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2976
2977     my $kvm_off = 0;
2978     my $gpu_passthrough;
2979
2980     # host pci devices
2981     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2982         my $d = parse_hostpci($conf->{"hostpci$i"});
2983         next if !$d;
2984
2985         my $pcie = $d->{pcie};
2986         if($pcie){
2987             die "q35 machine model is not enabled" if !$q35;
2988             $pciaddr = print_pcie_addr("hostpci$i");
2989         }else{
2990             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2991         }
2992
2993         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2994         my $romfile = $d->{romfile};
2995
2996         my $xvga = '';
2997         if ($d->{'x-vga'}) {
2998             $xvga = ',x-vga=on';
2999             $kvm_off = 1;
3000             $vga = 'none';
3001             $gpu_passthrough = 1;
3002
3003             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3004                 $xvga = "";
3005             }
3006         }
3007         my $pcidevices = $d->{pciid};
3008         my $multifunction = 1 if @$pcidevices > 1;
3009
3010         my $j=0;
3011         foreach my $pcidevice (@$pcidevices) {
3012
3013             my $id = "hostpci$i";
3014             $id .= ".$j" if $multifunction;
3015             my $addr = $pciaddr;
3016             $addr .= ".$j" if $multifunction;
3017             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3018
3019             if($j == 0){
3020                 $devicestr .= "$rombar$xvga";
3021                 $devicestr .= ",multifunction=on" if $multifunction;
3022                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3023             }
3024
3025             push @$devices, '-device', $devicestr;
3026             $j++;
3027         }
3028     }
3029
3030     # usb devices
3031     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3032     push @$devices, @usbdevices if @usbdevices;
3033     # serial devices
3034     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3035         if (my $path = $conf->{"serial$i"}) {
3036             if ($path eq 'socket') {
3037                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3038                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3039                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3040             } else {
3041                 die "no such serial device\n" if ! -c $path;
3042                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3043                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3044             }
3045         }
3046     }
3047
3048     # parallel devices
3049     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3050         if (my $path = $conf->{"parallel$i"}) {
3051             die "no such parallel device\n" if ! -c $path;
3052             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3053             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3054             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3055         }
3056     }
3057
3058     my $vmname = $conf->{name} || "vm$vmid";
3059
3060     push @$cmd, '-name', $vmname;
3061
3062     my $sockets = 1;
3063     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3064     $sockets = $conf->{sockets} if  $conf->{sockets};
3065
3066     my $cores = $conf->{cores} || 1;
3067
3068     my $maxcpus = $sockets * $cores;
3069
3070     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3071
3072     my $allowed_vcpus = $cpuinfo->{cpus};
3073
3074     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3075         if ($allowed_vcpus < $maxcpus);
3076
3077     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3078
3079         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3080         for (my $i = 2; $i <= $vcpus; $i++)  {
3081             my $cpustr = print_cpu_device($conf,$i);
3082             push @$cmd, '-device', $cpustr;
3083         }
3084
3085     } else {
3086
3087         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3088     }
3089     push @$cmd, '-nodefaults';
3090
3091     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3092
3093     my $bootindex_hash = {};
3094     my $i = 1;
3095     foreach my $o (split(//, $bootorder)) {
3096         $bootindex_hash->{$o} = $i*100;
3097         $i++;
3098     }
3099
3100     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3101
3102     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3103
3104     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3105
3106     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3107
3108     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3109         my $socket = vnc_socket($vmid);
3110         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3111     } else {
3112         push @$cmd, '-nographic';
3113     }
3114
3115     # time drift fix
3116     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3117
3118     my $useLocaltime = $conf->{localtime};
3119
3120     if ($winversion >= 5) { # windows
3121         $useLocaltime = 1 if !defined($conf->{localtime});
3122
3123         # use time drift fix when acpi is enabled
3124         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3125             $tdf = 1 if !defined($conf->{tdf});
3126         }
3127     }
3128
3129     if ($winversion >= 6) {
3130         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3131         push @$cmd, '-no-hpet';
3132     }
3133
3134     push @$rtcFlags, 'driftfix=slew' if $tdf;
3135
3136     if (!$kvm) {
3137         push @$machineFlags, 'accel=tcg';
3138     }
3139
3140     if ($machine_type) {
3141         push @$machineFlags, "type=${machine_type}";
3142     }
3143
3144     if ($conf->{startdate}) {
3145         push @$rtcFlags, "base=$conf->{startdate}";
3146     } elsif ($useLocaltime) {
3147         push @$rtcFlags, 'base=localtime';
3148     }
3149
3150     my $cpu = $kvm ? "kvm64" : "qemu64";
3151     if (my $cputype = $conf->{cpu}) {
3152         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3153             or die "Cannot parse cpu description: $cputype\n";
3154         $cpu = $cpuconf->{cputype};
3155         $kvm_off = 1 if $cpuconf->{hidden};
3156
3157         if (defined(my $flags = $cpuconf->{flags})) {
3158             push @$cpuFlags, $flags;
3159         }
3160     }
3161
3162     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3163
3164     push @$cpuFlags , '-x2apic'
3165         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3166
3167     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3168
3169     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3170
3171     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3172
3173         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3174         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3175     }
3176
3177     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3178
3179     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3180
3181     push @$cpuFlags, 'kvm=off' if $kvm_off;
3182
3183     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3184         die "internal error"; # should not happen
3185
3186     push @$cpuFlags, "vendor=${cpu_vendor}"
3187         if $cpu_vendor ne 'default';
3188
3189     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3190
3191     push @$cmd, '-cpu', $cpu;
3192
3193     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3194
3195     push @$cmd, '-S' if $conf->{freeze};
3196
3197     # set keyboard layout
3198     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3199     push @$cmd, '-k', $kb if $kb;
3200
3201     # enable sound
3202     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3203     #push @$cmd, '-soundhw', 'es1370';
3204     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3205
3206     if($conf->{agent}) {
3207         my $qgasocket = qmp_socket($vmid, 1);
3208         my $pciaddr = print_pci_addr("qga0", $bridges);
3209         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3210         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3211         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3212     }
3213
3214     my $spice_port;
3215
3216     if ($qxlnum) {
3217         if ($qxlnum > 1) {
3218             if ($winversion){
3219                 for(my $i = 1; $i < $qxlnum; $i++){
3220                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3221                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3222                 }
3223             } else {
3224                 # assume other OS works like Linux
3225                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3226                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3227             }
3228         }
3229
3230         my $pciaddr = print_pci_addr("spice", $bridges);
3231
3232         my $nodename = PVE::INotify::nodename();
3233         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3234         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3235         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3236         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3237         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3238
3239         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3240
3241         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3242         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3243         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3244     }
3245
3246     # enable balloon by default, unless explicitly disabled
3247     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3248         $pciaddr = print_pci_addr("balloon0", $bridges);
3249         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3250     }
3251
3252     if ($conf->{watchdog}) {
3253         my $wdopts = parse_watchdog($conf->{watchdog});
3254         $pciaddr = print_pci_addr("watchdog", $bridges);
3255         my $watchdog = $wdopts->{model} || 'i6300esb';
3256         push @$devices, '-device', "$watchdog$pciaddr";
3257         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3258     }
3259
3260     my $vollist = [];
3261     my $scsicontroller = {};
3262     my $ahcicontroller = {};
3263     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3264
3265     # Add iscsi initiator name if available
3266     if (my $initiator = get_initiator_name()) {
3267         push @$devices, '-iscsi', "initiator-name=$initiator";
3268     }
3269
3270     foreach_drive($conf, sub {
3271         my ($ds, $drive) = @_;
3272
3273         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3274             push @$vollist, $drive->{file};
3275         }
3276
3277         # ignore efidisk here, already added in bios/fw handling code above
3278         return if $drive->{interface} eq 'efidisk';
3279
3280         $use_virtio = 1 if $ds =~ m/^virtio/;
3281
3282         if (drive_is_cdrom ($drive)) {
3283             if ($bootindex_hash->{d}) {
3284                 $drive->{bootindex} = $bootindex_hash->{d};
3285                 $bootindex_hash->{d} += 1;
3286             }
3287         } else {
3288             if ($bootindex_hash->{c}) {
3289                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3290                 $bootindex_hash->{c} += 1;
3291             }
3292         }
3293
3294         if($drive->{interface} eq 'virtio'){
3295            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3296         }
3297
3298         if ($drive->{interface} eq 'scsi') {
3299
3300             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3301
3302             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3303             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3304
3305             my $iothread = '';
3306             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3307                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3308                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3309             } elsif ($drive->{iothread}) {
3310                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3311             }
3312
3313             my $queues = '';
3314             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3315                 $queues = ",num_queues=$drive->{queues}";
3316             }
3317
3318             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3319             $scsicontroller->{$controller}=1;
3320         }
3321
3322         if ($drive->{interface} eq 'sata') {
3323            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3324            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3325            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3326            $ahcicontroller->{$controller}=1;
3327         }
3328
3329         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3330         push @$devices, '-drive',$drive_cmd;
3331         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3332     });
3333
3334     for (my $i = 0; $i < $MAX_NETS; $i++) {
3335          next if !$conf->{"net$i"};
3336          my $d = parse_net($conf->{"net$i"});
3337          next if !$d;
3338
3339          $use_virtio = 1 if $d->{model} eq 'virtio';
3340
3341          if ($bootindex_hash->{n}) {
3342             $d->{bootindex} = $bootindex_hash->{n};
3343             $bootindex_hash->{n} += 1;
3344          }
3345
3346          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3347          push @$devices, '-netdev', $netdevfull;
3348
3349          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3350          push @$devices, '-device', $netdevicefull;
3351     }
3352
3353     if (!$q35) {
3354         # add pci bridges
3355         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3356            $bridges->{1} = 1;
3357            $bridges->{2} = 1;
3358         }
3359
3360         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3361
3362         while (my ($k, $v) = each %$bridges) {
3363             $pciaddr = print_pci_addr("pci.$k");
3364             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3365         }
3366     }
3367
3368     # add custom args
3369     if ($conf->{args}) {
3370         my $aa = PVE::Tools::split_args($conf->{args});
3371         push @$cmd, @$aa;
3372     }
3373
3374     push @$cmd, @$devices;
3375     push @$cmd, '-rtc', join(',', @$rtcFlags)
3376         if scalar(@$rtcFlags);
3377     push @$cmd, '-machine', join(',', @$machineFlags)
3378         if scalar(@$machineFlags);
3379     push @$cmd, '-global', join(',', @$globalFlags)
3380         if scalar(@$globalFlags);
3381
3382     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3383 }
3384
3385 sub vnc_socket {
3386     my ($vmid) = @_;
3387     return "${var_run_tmpdir}/$vmid.vnc";
3388 }
3389
3390 sub spice_port {
3391     my ($vmid) = @_;
3392
3393     my $res = vm_mon_cmd($vmid, 'query-spice');
3394
3395     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3396 }
3397
3398 sub qmp_socket {
3399     my ($vmid, $qga) = @_;
3400     my $sockettype = $qga ? 'qga' : 'qmp';
3401     return "${var_run_tmpdir}/$vmid.$sockettype";
3402 }
3403
3404 sub pidfile_name {
3405     my ($vmid) = @_;
3406     return "${var_run_tmpdir}/$vmid.pid";
3407 }
3408
3409 sub vm_devices_list {
3410     my ($vmid) = @_;
3411
3412     my $res = vm_mon_cmd($vmid, 'query-pci');
3413     my $devices = {};
3414     foreach my $pcibus (@$res) {
3415         foreach my $device (@{$pcibus->{devices}}) {
3416             next if !$device->{'qdev_id'};
3417             if ($device->{'pci_bridge'}) {
3418                 $devices->{$device->{'qdev_id'}} = 1;
3419                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3420                     next if !$bridge_device->{'qdev_id'};
3421                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3422                     $devices->{$device->{'qdev_id'}}++;
3423                 }
3424             } else {
3425                 $devices->{$device->{'qdev_id'}} = 1;
3426             }
3427         }
3428     }
3429
3430     my $resblock = vm_mon_cmd($vmid, 'query-block');
3431     foreach my $block (@$resblock) {
3432         if($block->{device} =~ m/^drive-(\S+)/){
3433                 $devices->{$1} = 1;
3434         }
3435     }
3436
3437     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3438     foreach my $mice (@$resmice) {
3439         if ($mice->{name} eq 'QEMU HID Tablet') {
3440             $devices->{tablet} = 1;
3441             last;
3442         }
3443     }
3444
3445     # for usb devices there is no query-usb
3446     # but we can iterate over the entries in
3447     # qom-list path=/machine/peripheral
3448     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3449     foreach my $per (@$resperipheral) {
3450         if ($per->{name} =~ m/^usb\d+$/) {
3451             $devices->{$per->{name}} = 1;
3452         }
3453     }
3454
3455     return $devices;
3456 }
3457
3458 sub vm_deviceplug {
3459     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3460
3461     my $q35 = machine_type_is_q35($conf);
3462
3463     my $devices_list = vm_devices_list($vmid);
3464     return 1 if defined($devices_list->{$deviceid});
3465
3466     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3467
3468     if ($deviceid eq 'tablet') {
3469
3470         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3471
3472     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3473
3474         die "usb hotplug currently not reliable\n";
3475         # since we can't reliably hot unplug all added usb devices
3476         # and usb passthrough disables live migration
3477         # we disable usb hotplugging for now
3478         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3479
3480     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3481
3482         qemu_iothread_add($vmid, $deviceid, $device);
3483
3484         qemu_driveadd($storecfg, $vmid, $device);
3485         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3486
3487         qemu_deviceadd($vmid, $devicefull);
3488         eval { qemu_deviceaddverify($vmid, $deviceid); };
3489         if (my $err = $@) {
3490             eval { qemu_drivedel($vmid, $deviceid); };
3491             warn $@ if $@;
3492             die $err;
3493         }
3494
3495     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3496
3497
3498         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3499         my $pciaddr = print_pci_addr($deviceid);
3500         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3501
3502         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3503
3504         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3505             qemu_iothread_add($vmid, $deviceid, $device);
3506             $devicefull .= ",iothread=iothread-$deviceid";
3507         }
3508
3509         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3510             $devicefull .= ",num_queues=$device->{queues}";
3511         }
3512
3513         qemu_deviceadd($vmid, $devicefull);
3514         qemu_deviceaddverify($vmid, $deviceid);
3515
3516     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3517
3518         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3519         qemu_driveadd($storecfg, $vmid, $device);
3520
3521         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3522         eval { qemu_deviceadd($vmid, $devicefull); };
3523         if (my $err = $@) {
3524             eval { qemu_drivedel($vmid, $deviceid); };
3525             warn $@ if $@;
3526             die $err;
3527         }
3528
3529     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3530
3531         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3532
3533         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3534         my $use_old_bios_files = undef;
3535         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3536
3537         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3538         qemu_deviceadd($vmid, $netdevicefull);
3539         eval { qemu_deviceaddverify($vmid, $deviceid); };
3540         if (my $err = $@) {
3541             eval { qemu_netdevdel($vmid, $deviceid); };
3542             warn $@ if $@;
3543             die $err;
3544         }
3545
3546     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3547
3548         my $bridgeid = $2;
3549         my $pciaddr = print_pci_addr($deviceid);
3550         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3551
3552         qemu_deviceadd($vmid, $devicefull);
3553         qemu_deviceaddverify($vmid, $deviceid);
3554
3555     } else {
3556         die "can't hotplug device '$deviceid'\n";
3557     }
3558
3559     return 1;
3560 }
3561
3562 # fixme: this should raise exceptions on error!
3563 sub vm_deviceunplug {
3564     my ($vmid, $conf, $deviceid) = @_;
3565
3566     my $devices_list = vm_devices_list($vmid);
3567     return 1 if !defined($devices_list->{$deviceid});
3568
3569     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3570
3571     if ($deviceid eq 'tablet') {
3572
3573         qemu_devicedel($vmid, $deviceid);
3574
3575     } elsif ($deviceid =~ m/^usb\d+$/) {
3576
3577         die "usb hotplug currently not reliable\n";
3578         # when unplugging usb devices this way,
3579         # there may be remaining usb controllers/hubs
3580         # so we disable it for now
3581         qemu_devicedel($vmid, $deviceid);
3582         qemu_devicedelverify($vmid, $deviceid);
3583
3584     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3585
3586         qemu_devicedel($vmid, $deviceid);
3587         qemu_devicedelverify($vmid, $deviceid);
3588         qemu_drivedel($vmid, $deviceid);
3589         qemu_iothread_del($conf, $vmid, $deviceid);
3590
3591     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3592
3593         qemu_devicedel($vmid, $deviceid);
3594         qemu_devicedelverify($vmid, $deviceid);
3595         qemu_iothread_del($conf, $vmid, $deviceid);
3596
3597     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3598
3599         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3600         my $device = parse_drive($deviceid, $conf->{$deviceid});
3601         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3602
3603         qemu_devicedel($vmid, $deviceid);
3604         qemu_drivedel($vmid, $deviceid);
3605         qemu_deletescsihw($conf, $vmid, $deviceid);
3606
3607     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3608
3609         qemu_devicedel($vmid, $deviceid);
3610         qemu_devicedelverify($vmid, $deviceid);
3611         qemu_netdevdel($vmid, $deviceid);
3612
3613     } else {
3614         die "can't unplug device '$deviceid'\n";
3615     }
3616
3617     return 1;
3618 }
3619
3620 sub qemu_deviceadd {
3621     my ($vmid, $devicefull) = @_;
3622
3623     $devicefull = "driver=".$devicefull;
3624     my %options =  split(/[=,]/, $devicefull);
3625
3626     vm_mon_cmd($vmid, "device_add" , %options);
3627 }
3628
3629 sub qemu_devicedel {
3630     my ($vmid, $deviceid) = @_;
3631
3632     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3633 }
3634
3635 sub qemu_iothread_add {
3636     my($vmid, $deviceid, $device) = @_;
3637
3638     if ($device->{iothread}) {
3639         my $iothreads = vm_iothreads_list($vmid);
3640         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3641     }
3642 }
3643
3644 sub qemu_iothread_del {
3645     my($conf, $vmid, $deviceid) = @_;
3646
3647     my $device = parse_drive($deviceid, $conf->{$deviceid});
3648     if ($device->{iothread}) {
3649         my $iothreads = vm_iothreads_list($vmid);
3650         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3651     }
3652 }
3653
3654 sub qemu_objectadd {
3655     my($vmid, $objectid, $qomtype) = @_;
3656
3657     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3658
3659     return 1;
3660 }
3661
3662 sub qemu_objectdel {
3663     my($vmid, $objectid) = @_;
3664
3665     vm_mon_cmd($vmid, "object-del", id => $objectid);
3666
3667     return 1;
3668 }
3669
3670 sub qemu_driveadd {
3671     my ($storecfg, $vmid, $device) = @_;
3672
3673     my $drive = print_drive_full($storecfg, $vmid, $device);
3674     $drive =~ s/\\/\\\\/g;
3675     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3676
3677     # If the command succeeds qemu prints: "OK"
3678     return 1 if $ret =~ m/OK/s;
3679
3680     die "adding drive failed: $ret\n";
3681 }
3682
3683 sub qemu_drivedel {
3684     my($vmid, $deviceid) = @_;
3685
3686     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3687     $ret =~ s/^\s+//;
3688
3689     return 1 if $ret eq "";
3690
3691     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3692     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3693
3694     die "deleting drive $deviceid failed : $ret\n";
3695 }
3696
3697 sub qemu_deviceaddverify {
3698     my ($vmid, $deviceid) = @_;
3699
3700     for (my $i = 0; $i <= 5; $i++) {
3701          my $devices_list = vm_devices_list($vmid);
3702          return 1 if defined($devices_list->{$deviceid});
3703          sleep 1;
3704     }
3705
3706     die "error on hotplug device '$deviceid'\n";
3707 }
3708
3709
3710 sub qemu_devicedelverify {
3711     my ($vmid, $deviceid) = @_;
3712
3713     # need to verify that the device is correctly removed as device_del
3714     # is async and empty return is not reliable
3715
3716     for (my $i = 0; $i <= 5; $i++) {
3717          my $devices_list = vm_devices_list($vmid);
3718          return 1 if !defined($devices_list->{$deviceid});
3719          sleep 1;
3720     }
3721
3722     die "error on hot-unplugging device '$deviceid'\n";
3723 }
3724
3725 sub qemu_findorcreatescsihw {
3726     my ($storecfg, $conf, $vmid, $device) = @_;
3727
3728     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3729
3730     my $scsihwid="$controller_prefix$controller";
3731     my $devices_list = vm_devices_list($vmid);
3732
3733     if(!defined($devices_list->{$scsihwid})) {
3734         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3735     }
3736
3737     return 1;
3738 }
3739
3740 sub qemu_deletescsihw {
3741     my ($conf, $vmid, $opt) = @_;
3742
3743     my $device = parse_drive($opt, $conf->{$opt});
3744
3745     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3746         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3747         return 1;
3748     }
3749
3750     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3751
3752     my $devices_list = vm_devices_list($vmid);
3753     foreach my $opt (keys %{$devices_list}) {
3754         if (PVE::QemuServer::is_valid_drivename($opt)) {
3755             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3756             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3757                 return 1;
3758             }
3759         }
3760     }
3761
3762     my $scsihwid="scsihw$controller";
3763
3764     vm_deviceunplug($vmid, $conf, $scsihwid);
3765
3766     return 1;
3767 }
3768
3769 sub qemu_add_pci_bridge {
3770     my ($storecfg, $conf, $vmid, $device) = @_;
3771
3772     my $bridges = {};
3773
3774     my $bridgeid;
3775
3776     print_pci_addr($device, $bridges);
3777
3778     while (my ($k, $v) = each %$bridges) {
3779         $bridgeid = $k;
3780     }
3781     return 1 if !defined($bridgeid) || $bridgeid < 1;
3782
3783     my $bridge = "pci.$bridgeid";
3784     my $devices_list = vm_devices_list($vmid);
3785
3786     if (!defined($devices_list->{$bridge})) {
3787         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3788     }
3789
3790     return 1;
3791 }
3792
3793 sub qemu_set_link_status {
3794     my ($vmid, $device, $up) = @_;
3795
3796     vm_mon_cmd($vmid, "set_link", name => $device,
3797                up => $up ? JSON::true : JSON::false);
3798 }
3799
3800 sub qemu_netdevadd {
3801     my ($vmid, $conf, $device, $deviceid) = @_;
3802
3803     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3804     my %options =  split(/[=,]/, $netdev);
3805
3806     vm_mon_cmd($vmid, "netdev_add",  %options);
3807     return 1;
3808 }
3809
3810 sub qemu_netdevdel {
3811     my ($vmid, $deviceid) = @_;
3812
3813     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3814 }
3815
3816 sub qemu_usb_hotplug {
3817     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3818
3819     return if !$device;
3820
3821     # remove the old one first
3822     vm_deviceunplug($vmid, $conf, $deviceid);
3823
3824     # check if xhci controller is necessary and available
3825     if ($device->{usb3}) {
3826
3827         my $devicelist = vm_devices_list($vmid);
3828
3829         if (!$devicelist->{xhci}) {
3830             my $pciaddr = print_pci_addr("xhci");
3831             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3832         }
3833     }
3834     my $d = parse_usb_device($device->{host});
3835     $d->{usb3} = $device->{usb3};
3836
3837     # add the new one
3838     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3839 }
3840
3841 sub qemu_cpu_hotplug {
3842     my ($vmid, $conf, $vcpus) = @_;
3843
3844     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3845
3846     my $sockets = 1;
3847     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3848     $sockets = $conf->{sockets} if  $conf->{sockets};
3849     my $cores = $conf->{cores} || 1;
3850     my $maxcpus = $sockets * $cores;
3851
3852     $vcpus = $maxcpus if !$vcpus;
3853
3854     die "you can't add more vcpus than maxcpus\n"
3855         if $vcpus > $maxcpus;
3856
3857     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3858
3859     if ($vcpus < $currentvcpus) {
3860
3861         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3862
3863             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3864                 qemu_devicedel($vmid, "cpu$i");
3865                 my $retry = 0;
3866                 my $currentrunningvcpus = undef;
3867                 while (1) {
3868                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3869                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3870                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3871                     $retry++;
3872                     sleep 1;
3873                 }
3874                 #update conf after each succesfull cpu unplug
3875                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3876                 PVE::QemuConfig->write_config($vmid, $conf);
3877             }
3878         } else {
3879             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3880         }
3881
3882         return;
3883     }
3884
3885     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3886     die "vcpus in running vm does not match its configuration\n"
3887         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3888
3889     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3890
3891         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3892             my $cpustr = print_cpu_device($conf, $i);
3893             qemu_deviceadd($vmid, $cpustr);
3894
3895             my $retry = 0;
3896             my $currentrunningvcpus = undef;
3897             while (1) {
3898                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3899                 last if scalar(@{$currentrunningvcpus}) == $i;
3900                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3901                 sleep 1;
3902                 $retry++;
3903             }
3904             #update conf after each succesfull cpu hotplug
3905             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3906             PVE::QemuConfig->write_config($vmid, $conf);
3907         }
3908     } else {
3909
3910         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3911             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3912         }
3913     }
3914 }
3915
3916 sub qemu_block_set_io_throttle {
3917     my ($vmid, $deviceid,
3918         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3919         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3920         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3921         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3922
3923     return if !check_running($vmid) ;
3924
3925     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3926         bps => int($bps),
3927         bps_rd => int($bps_rd),
3928         bps_wr => int($bps_wr),
3929         iops => int($iops),
3930         iops_rd => int($iops_rd),
3931         iops_wr => int($iops_wr),
3932         bps_max => int($bps_max),
3933         bps_rd_max => int($bps_rd_max),
3934         bps_wr_max => int($bps_wr_max),
3935         iops_max => int($iops_max),
3936         iops_rd_max => int($iops_rd_max),
3937         iops_wr_max => int($iops_wr_max),
3938         bps_max_length => int($bps_max_length),
3939         bps_rd_max_length => int($bps_rd_max_length),
3940         bps_wr_max_length => int($bps_wr_max_length),
3941         iops_max_length => int($iops_max_length),
3942         iops_rd_max_length => int($iops_rd_max_length),
3943         iops_wr_max_length => int($iops_wr_max_length),
3944     );
3945
3946 }
3947
3948 # old code, only used to shutdown old VM after update
3949 sub __read_avail {
3950     my ($fh, $timeout) = @_;
3951
3952     my $sel = new IO::Select;
3953     $sel->add($fh);
3954
3955     my $res = '';
3956     my $buf;
3957
3958     my @ready;
3959     while (scalar (@ready = $sel->can_read($timeout))) {
3960         my $count;
3961         if ($count = $fh->sysread($buf, 8192)) {
3962             if ($buf =~ /^(.*)\(qemu\) $/s) {
3963                 $res .= $1;
3964                 last;
3965             } else {
3966                 $res .= $buf;
3967             }
3968         } else {
3969             if (!defined($count)) {
3970                 die "$!\n";
3971             }
3972             last;
3973         }
3974     }
3975
3976     die "monitor read timeout\n" if !scalar(@ready);
3977
3978     return $res;
3979 }
3980
3981 # old code, only used to shutdown old VM after update
3982 sub vm_monitor_command {
3983     my ($vmid, $cmdstr, $nocheck) = @_;
3984
3985     my $res;
3986
3987     eval {
3988         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3989
3990         my $sname = "${var_run_tmpdir}/$vmid.mon";
3991
3992         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3993             die "unable to connect to VM $vmid socket - $!\n";
3994
3995         my $timeout = 3;
3996
3997         # hack: migrate sometime blocks the monitor (when migrate_downtime
3998         # is set)
3999         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4000             $timeout = 60*60; # 1 hour
4001         }
4002
4003         # read banner;
4004         my $data = __read_avail($sock, $timeout);
4005
4006         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
4007             die "got unexpected qemu monitor banner\n";
4008         }
4009
4010         my $sel = new IO::Select;
4011         $sel->add($sock);
4012
4013         if (!scalar(my @ready = $sel->can_write($timeout))) {
4014             die "monitor write error - timeout";
4015         }
4016
4017         my $fullcmd = "$cmdstr\r";
4018
4019         # syslog('info', "VM $vmid monitor command: $cmdstr");
4020
4021         my $b;
4022         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
4023             die "monitor write error - $!";
4024         }
4025
4026         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4027
4028         $timeout = 20;
4029
4030         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4031             $timeout = 60*60; # 1 hour
4032         } elsif ($cmdstr =~ m/^(eject|change)/) {
4033             $timeout = 60; # note: cdrom mount command is slow
4034         }
4035         if ($res = __read_avail($sock, $timeout)) {
4036
4037             my @lines = split("\r?\n", $res);
4038
4039             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4040
4041             $res = join("\n", @lines);
4042             $res .= "\n";
4043         }
4044     };
4045
4046     my $err = $@;
4047
4048     if ($err) {
4049         syslog("err", "VM $vmid monitor command failed - $err");
4050         die $err;
4051     }
4052
4053     return $res;
4054 }
4055
4056 sub qemu_block_resize {
4057     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4058
4059     my $running = check_running($vmid);
4060
4061     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4062
4063     return if !$running;
4064
4065     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4066
4067 }
4068
4069 sub qemu_volume_snapshot {
4070     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4071
4072     my $running = check_running($vmid);
4073
4074     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4075         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4076     } else {
4077         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4078     }
4079 }
4080
4081 sub qemu_volume_snapshot_delete {
4082     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4083
4084     my $running = check_running($vmid);
4085
4086     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4087         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4088     } else {
4089         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4090     }
4091 }
4092
4093 sub set_migration_caps {
4094     my ($vmid) = @_;
4095
4096     my $cap_ref = [];
4097
4098     my $enabled_cap = {
4099         "auto-converge" => 1,
4100         "xbzrle" => 1,
4101         "x-rdma-pin-all" => 0,
4102         "zero-blocks" => 0,
4103         "compress" => 0
4104     };
4105
4106     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4107
4108     for my $supported_capability (@$supported_capabilities) {
4109         push @$cap_ref, {
4110             capability => $supported_capability->{capability},
4111             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4112         };
4113     }
4114
4115     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4116 }
4117
4118 my $fast_plug_option = {
4119     'lock' => 1,
4120     'name' => 1,
4121     'onboot' => 1,
4122     'shares' => 1,
4123     'startup' => 1,
4124     'description' => 1,
4125     'protection' => 1,
4126     'vmstatestorage' => 1,
4127 };
4128
4129 # hotplug changes in [PENDING]
4130 # $selection hash can be used to only apply specified options, for
4131 # example: { cores => 1 } (only apply changed 'cores')
4132 # $errors ref is used to return error messages
4133 sub vmconfig_hotplug_pending {
4134     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4135
4136     my $defaults = load_defaults();
4137
4138     # commit values which do not have any impact on running VM first
4139     # Note: those option cannot raise errors, we we do not care about
4140     # $selection and always apply them.
4141
4142     my $add_error = sub {
4143         my ($opt, $msg) = @_;
4144         $errors->{$opt} = "hotplug problem - $msg";
4145     };
4146
4147     my $changes = 0;
4148     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4149         if ($fast_plug_option->{$opt}) {
4150             $conf->{$opt} = $conf->{pending}->{$opt};
4151             delete $conf->{pending}->{$opt};
4152             $changes = 1;
4153         }
4154     }
4155
4156     if ($changes) {
4157         PVE::QemuConfig->write_config($vmid, $conf);
4158         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4159     }
4160
4161     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4162
4163     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4164     while (my ($opt, $force) = each %$pending_delete_hash) {
4165         next if $selection && !$selection->{$opt};
4166         eval {
4167             if ($opt eq 'hotplug') {
4168                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4169             } elsif ($opt eq 'tablet') {
4170                 die "skip\n" if !$hotplug_features->{usb};
4171                 if ($defaults->{tablet}) {
4172                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4173                 } else {
4174                     vm_deviceunplug($vmid, $conf, $opt);
4175                 }
4176             } elsif ($opt =~ m/^usb\d+/) {
4177                 die "skip\n";
4178                 # since we cannot reliably hot unplug usb devices
4179                 # we are disabling it
4180                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4181                 vm_deviceunplug($vmid, $conf, $opt);
4182             } elsif ($opt eq 'vcpus') {
4183                 die "skip\n" if !$hotplug_features->{cpu};
4184                 qemu_cpu_hotplug($vmid, $conf, undef);
4185             } elsif ($opt eq 'balloon') {
4186                 # enable balloon device is not hotpluggable
4187                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4188             } elsif ($fast_plug_option->{$opt}) {
4189                 # do nothing
4190             } elsif ($opt =~ m/^net(\d+)$/) {
4191                 die "skip\n" if !$hotplug_features->{network};
4192                 vm_deviceunplug($vmid, $conf, $opt);
4193             } elsif (is_valid_drivename($opt)) {
4194                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4195                 vm_deviceunplug($vmid, $conf, $opt);
4196                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4197             } elsif ($opt =~ m/^memory$/) {
4198                 die "skip\n" if !$hotplug_features->{memory};
4199                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4200             } elsif ($opt eq 'cpuunits') {
4201                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4202             } elsif ($opt eq 'cpulimit') {
4203                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4204             } else {
4205                 die "skip\n";
4206             }
4207         };
4208         if (my $err = $@) {
4209             &$add_error($opt, $err) if $err ne "skip\n";
4210         } else {
4211             # save new config if hotplug was successful
4212             delete $conf->{$opt};
4213             vmconfig_undelete_pending_option($conf, $opt);
4214             PVE::QemuConfig->write_config($vmid, $conf);
4215             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4216         }
4217     }
4218
4219     foreach my $opt (keys %{$conf->{pending}}) {
4220         next if $selection && !$selection->{$opt};
4221         my $value = $conf->{pending}->{$opt};
4222         eval {
4223             if ($opt eq 'hotplug') {
4224                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4225             } elsif ($opt eq 'tablet') {
4226                 die "skip\n" if !$hotplug_features->{usb};
4227                 if ($value == 1) {
4228                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4229                 } elsif ($value == 0) {
4230                     vm_deviceunplug($vmid, $conf, $opt);
4231                 }
4232             } elsif ($opt =~ m/^usb\d+$/) {
4233                 die "skip\n";
4234                 # since we cannot reliably hot unplug usb devices
4235                 # we are disabling it
4236                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4237                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4238                 die "skip\n" if !$d;
4239                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4240             } elsif ($opt eq 'vcpus') {
4241                 die "skip\n" if !$hotplug_features->{cpu};
4242                 qemu_cpu_hotplug($vmid, $conf, $value);
4243             } elsif ($opt eq 'balloon') {
4244                 # enable/disable balloning device is not hotpluggable
4245                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4246                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4247                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4248
4249                 # allow manual ballooning if shares is set to zero
4250                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4251                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4252                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4253                 }
4254             } elsif ($opt =~ m/^net(\d+)$/) {
4255                 # some changes can be done without hotplug
4256                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4257                                     $vmid, $opt, $value);
4258             } elsif (is_valid_drivename($opt)) {
4259                 # some changes can be done without hotplug
4260                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4261                                      $vmid, $opt, $value, 1);
4262             } elsif ($opt =~ m/^memory$/) { #dimms
4263                 die "skip\n" if !$hotplug_features->{memory};
4264                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4265             } elsif ($opt eq 'cpuunits') {
4266                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4267             } elsif ($opt eq 'cpulimit') {
4268                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4269                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4270             } else {
4271                 die "skip\n";  # skip non-hot-pluggable options
4272             }
4273         };
4274         if (my $err = $@) {
4275             &$add_error($opt, $err) if $err ne "skip\n";
4276         } else {
4277             # save new config if hotplug was successful
4278             $conf->{$opt} = $value;
4279             delete $conf->{pending}->{$opt};
4280             PVE::QemuConfig->write_config($vmid, $conf);
4281             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4282         }
4283     }
4284 }
4285
4286 sub try_deallocate_drive {
4287     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4288
4289     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4290         my $volid = $drive->{file};
4291         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4292             my $sid = PVE::Storage::parse_volume_id($volid);
4293             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4294
4295             # check if the disk is really unused
4296             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4297                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4298             PVE::Storage::vdisk_free($storecfg, $volid);
4299             return 1;
4300         } else {
4301             # If vm is not owner of this disk remove from config
4302             return 1;
4303         }
4304     }
4305
4306     return undef;
4307 }
4308
4309 sub vmconfig_delete_or_detach_drive {
4310     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4311
4312     my $drive = parse_drive($opt, $conf->{$opt});
4313
4314     my $rpcenv = PVE::RPCEnvironment::get();
4315     my $authuser = $rpcenv->get_user();
4316
4317     if ($force) {
4318         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4319         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4320     } else {
4321         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4322     }
4323 }
4324
4325 sub vmconfig_apply_pending {
4326     my ($vmid, $conf, $storecfg) = @_;
4327
4328     # cold plug
4329
4330     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4331     while (my ($opt, $force) = each %$pending_delete_hash) {
4332         die "internal error" if $opt =~ m/^unused/;
4333         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4334         if (!defined($conf->{$opt})) {
4335             vmconfig_undelete_pending_option($conf, $opt);
4336             PVE::QemuConfig->write_config($vmid, $conf);
4337         } elsif (is_valid_drivename($opt)) {
4338             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4339             vmconfig_undelete_pending_option($conf, $opt);
4340             delete $conf->{$opt};
4341             PVE::QemuConfig->write_config($vmid, $conf);
4342         } else {
4343             vmconfig_undelete_pending_option($conf, $opt);
4344             delete $conf->{$opt};
4345             PVE::QemuConfig->write_config($vmid, $conf);
4346         }
4347     }
4348
4349     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4350
4351     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4352         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4353
4354         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4355             # skip if nothing changed
4356         } elsif (is_valid_drivename($opt)) {
4357             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4358                 if defined($conf->{$opt});
4359             $conf->{$opt} = $conf->{pending}->{$opt};
4360         } else {
4361             $conf->{$opt} = $conf->{pending}->{$opt};
4362         }
4363
4364         delete $conf->{pending}->{$opt};
4365         PVE::QemuConfig->write_config($vmid, $conf);
4366     }
4367 }
4368
4369 my $safe_num_ne = sub {
4370     my ($a, $b) = @_;
4371
4372     return 0 if !defined($a) && !defined($b);
4373     return 1 if !defined($a);
4374     return 1 if !defined($b);
4375
4376     return $a != $b;
4377 };
4378
4379 my $safe_string_ne = sub {
4380     my ($a, $b) = @_;
4381
4382     return 0 if !defined($a) && !defined($b);
4383     return 1 if !defined($a);
4384     return 1 if !defined($b);
4385
4386     return $a ne $b;
4387 };
4388
4389 sub vmconfig_update_net {
4390     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4391
4392     my $newnet = parse_net($value);
4393
4394     if ($conf->{$opt}) {
4395         my $oldnet = parse_net($conf->{$opt});
4396
4397         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4398             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4399             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4400             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4401
4402             # for non online change, we try to hot-unplug
4403             die "skip\n" if !$hotplug;
4404             vm_deviceunplug($vmid, $conf, $opt);
4405         } else {
4406
4407             die "internal error" if $opt !~ m/net(\d+)/;
4408             my $iface = "tap${vmid}i$1";
4409
4410             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4411                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4412                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4413                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4414                 PVE::Network::tap_unplug($iface);
4415                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4416             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4417                 # Rate can be applied on its own but any change above needs to
4418                 # include the rate in tap_plug since OVS resets everything.
4419                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4420             }
4421
4422             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4423                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4424             }
4425
4426             return 1;
4427         }
4428     }
4429
4430     if ($hotplug) {
4431         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4432     } else {
4433         die "skip\n";
4434     }
4435 }
4436
4437 sub vmconfig_update_disk {
4438     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4439
4440     # fixme: do we need force?
4441
4442     my $drive = parse_drive($opt, $value);
4443
4444     if ($conf->{$opt}) {
4445
4446         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4447
4448             my $media = $drive->{media} || 'disk';
4449             my $oldmedia = $old_drive->{media} || 'disk';
4450             die "unable to change media type\n" if $media ne $oldmedia;
4451
4452             if (!drive_is_cdrom($old_drive)) {
4453
4454                 if ($drive->{file} ne $old_drive->{file}) {
4455
4456                     die "skip\n" if !$hotplug;
4457
4458                     # unplug and register as unused
4459                     vm_deviceunplug($vmid, $conf, $opt);
4460                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4461
4462                 } else {
4463                     # update existing disk
4464
4465                     # skip non hotpluggable value
4466                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4467                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4468                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4469                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4470                         die "skip\n";
4471                     }
4472
4473                     # apply throttle
4474                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4475                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4476                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4477                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4478                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4479                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4480                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4481                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4482                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4483                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4484                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4485                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4486                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4487                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4488                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4489                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4490                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4491                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4492
4493                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4494                                                    ($drive->{mbps} || 0)*1024*1024,
4495                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4496                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4497                                                    $drive->{iops} || 0,
4498                                                    $drive->{iops_rd} || 0,
4499                                                    $drive->{iops_wr} || 0,
4500                                                    ($drive->{mbps_max} || 0)*1024*1024,
4501                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4502                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4503                                                    $drive->{iops_max} || 0,
4504                                                    $drive->{iops_rd_max} || 0,
4505                                                    $drive->{iops_wr_max} || 0,
4506                                                    $drive->{bps_max_length} || 1,
4507                                                    $drive->{bps_rd_max_length} || 1,
4508                                                    $drive->{bps_wr_max_length} || 1,
4509                                                    $drive->{iops_max_length} || 1,
4510                                                    $drive->{iops_rd_max_length} || 1,
4511                                                    $drive->{iops_wr_max_length} || 1);
4512
4513                     }
4514
4515                     return 1;
4516                 }
4517
4518             } else { # cdrom
4519
4520                 if ($drive->{file} eq 'none') {
4521                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4522                 } else {
4523                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4524                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4525                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4526                 }
4527
4528                 return 1;
4529             }
4530         }
4531     }
4532
4533     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4534     # hotplug new disks
4535     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4536     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4537 }
4538
4539 sub vm_start {
4540     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4541         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4542
4543     PVE::QemuConfig->lock_config($vmid, sub {
4544         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4545
4546         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4547
4548         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4549
4550         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4551
4552         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4553             vmconfig_apply_pending($vmid, $conf, $storecfg);
4554             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4555         }
4556
4557         my $defaults = load_defaults();
4558
4559         # set environment variable useful inside network script
4560         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4561
4562         my $local_volumes = {};
4563
4564         if ($targetstorage) {
4565             foreach_drive($conf, sub {
4566                 my ($ds, $drive) = @_;
4567
4568                 return if drive_is_cdrom($drive);
4569
4570                 my $volid = $drive->{file};
4571
4572                 return if !$volid;
4573
4574                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4575
4576                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4577                 return if $scfg->{shared};
4578                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4579             });
4580
4581             my $format = undef;
4582
4583             foreach my $opt (sort keys %$local_volumes) {
4584
4585                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4586                 my $drive = parse_drive($opt, $conf->{$opt});
4587
4588                 #if remote storage is specified, use default format
4589                 if ($targetstorage && $targetstorage ne "1") {
4590                     $storeid = $targetstorage;
4591                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4592                     $format = $defFormat;
4593                 } else {
4594                     #else we use same format than original
4595                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4596                     $format = qemu_img_format($scfg, $volid);
4597                 }
4598
4599                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4600                 my $newdrive = $drive;
4601                 $newdrive->{format} = $format;
4602                 $newdrive->{file} = $newvolid;
4603                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4604                 $local_volumes->{$opt} = $drivestr;
4605                 #pass drive to conf for command line
4606                 $conf->{$opt} = $drivestr;
4607             }
4608         }
4609
4610         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4611
4612         my $migrate_port = 0;
4613         my $migrate_uri;
4614         if ($statefile) {
4615             if ($statefile eq 'tcp') {
4616                 my $localip = "localhost";
4617                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4618                 my $nodename = PVE::INotify::nodename();
4619
4620                 if (!defined($migration_type)) {
4621                     if (defined($datacenterconf->{migration}->{type})) {
4622                         $migration_type = $datacenterconf->{migration}->{type};
4623                     } else {
4624                         $migration_type = 'secure';
4625                     }
4626                 }
4627
4628                 if ($migration_type eq 'insecure') {
4629                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4630                     if ($migrate_network_addr) {
4631                         $localip = $migrate_network_addr;
4632                     } else {
4633                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4634                     }
4635
4636                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4637                 }
4638
4639                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4640                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4641                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4642                 push @$cmd, '-incoming', $migrate_uri;
4643                 push @$cmd, '-S';
4644
4645             } elsif ($statefile eq 'unix') {
4646                 # should be default for secure migrations as a ssh TCP forward
4647                 # tunnel is not deterministic reliable ready and fails regurarly
4648                 # to set up in time, so use UNIX socket forwards
4649                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4650                 unlink $socket_addr;
4651
4652                 $migrate_uri = "unix:$socket_addr";
4653
4654                 push @$cmd, '-incoming', $migrate_uri;
4655                 push @$cmd, '-S';
4656
4657             } else {
4658                 push @$cmd, '-loadstate', $statefile;
4659             }
4660         } elsif ($paused) {
4661             push @$cmd, '-S';
4662         }
4663
4664         # host pci devices
4665         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4666           my $d = parse_hostpci($conf->{"hostpci$i"});
4667           next if !$d;
4668           my $pcidevices = $d->{pciid};
4669           foreach my $pcidevice (@$pcidevices) {
4670                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4671
4672                 my $info = pci_device_info("0000:$pciid");
4673                 die "IOMMU not present\n" if !check_iommu_support();
4674                 die "no pci device info for device '$pciid'\n" if !$info;
4675                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4676                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4677           }
4678         }
4679
4680         PVE::Storage::activate_volumes($storecfg, $vollist);
4681
4682         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4683             my $cmd = [];
4684             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4685             eval  { run_command($cmd); };
4686         }
4687
4688         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4689                                                   : $defaults->{cpuunits};
4690
4691         my %run_params = (timeout => $statefile ? undef : 30, umask => 0077);
4692
4693         my %properties = (
4694             Slice => 'qemu.slice',
4695             KillMode => 'none',
4696             CPUShares => $cpuunits
4697         );
4698
4699         if (my $cpulimit = $conf->{cpulimit}) {
4700             $properties{CPUQuota} = int($cpulimit * 100);
4701         }
4702         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4703
4704         if ($conf->{hugepages}) {
4705
4706             my $code = sub {
4707                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4708                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4709
4710                 PVE::QemuServer::Memory::hugepages_mount();
4711                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4712
4713                 eval  {
4714                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4715                     run_command($cmd, %run_params);
4716                 };
4717
4718                 if (my $err = $@) {
4719                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4720                     die $err;
4721                 }
4722
4723                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4724             };
4725             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4726
4727         } else {
4728             eval  {
4729                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4730                 run_command($cmd, %run_params);
4731             };
4732         }
4733
4734         if (my $err = $@) {
4735             # deactivate volumes if start fails
4736             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4737             die "start failed: $err";
4738         }
4739
4740         print "migration listens on $migrate_uri\n" if $migrate_uri;
4741
4742         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4743             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4744             warn $@ if $@;
4745         }
4746
4747         #start nbd server for storage migration
4748         if ($targetstorage) {
4749             my $nodename = PVE::INotify::nodename();
4750             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4751             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4752             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4753             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4754
4755             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4756
4757             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4758
4759             foreach my $opt (sort keys %$local_volumes) {
4760                 my $volid = $local_volumes->{$opt};
4761                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4762                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4763                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4764             }
4765         }
4766
4767         if ($migratedfrom) {
4768             eval {
4769                 set_migration_caps($vmid);
4770             };
4771             warn $@ if $@;
4772
4773             if ($spice_port) {
4774                 print "spice listens on port $spice_port\n";
4775                 if ($spice_ticket) {
4776                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4777                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4778                 }
4779             }
4780
4781         } else {
4782             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4783                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4784                     if $conf->{balloon};
4785             }
4786
4787             foreach my $opt (keys %$conf) {
4788                 next if $opt !~  m/^net\d+$/;
4789                 my $nicconf = parse_net($conf->{$opt});
4790                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4791             }
4792         }
4793
4794         vm_mon_cmd_nocheck($vmid, 'qom-set',
4795                     path => "machine/peripheral/balloon0",
4796                     property => "guest-stats-polling-interval",
4797                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4798
4799     });
4800 }
4801
4802 sub vm_mon_cmd {
4803     my ($vmid, $execute, %params) = @_;
4804
4805     my $cmd = { execute => $execute, arguments => \%params };
4806     vm_qmp_command($vmid, $cmd);
4807 }
4808
4809 sub vm_mon_cmd_nocheck {
4810     my ($vmid, $execute, %params) = @_;
4811
4812     my $cmd = { execute => $execute, arguments => \%params };
4813     vm_qmp_command($vmid, $cmd, 1);
4814 }
4815
4816 sub vm_qmp_command {
4817     my ($vmid, $cmd, $nocheck) = @_;
4818
4819     my $res;
4820
4821     my $timeout;
4822     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4823         $timeout = $cmd->{arguments}->{timeout};
4824         delete $cmd->{arguments}->{timeout};
4825     }
4826
4827     eval {
4828         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4829         my $sname = qmp_socket($vmid);
4830         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4831             my $qmpclient = PVE::QMPClient->new();
4832
4833             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4834         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4835             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4836                 if scalar(%{$cmd->{arguments}});
4837             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4838         } else {
4839             die "unable to open monitor socket\n";
4840         }
4841     };
4842     if (my $err = $@) {
4843         syslog("err", "VM $vmid qmp command failed - $err");
4844         die $err;
4845     }
4846
4847     return $res;
4848 }
4849
4850 sub vm_human_monitor_command {
4851     my ($vmid, $cmdline) = @_;
4852
4853     my $res;
4854
4855     my $cmd = {
4856         execute => 'human-monitor-command',
4857         arguments => { 'command-line' => $cmdline},
4858     };
4859
4860     return vm_qmp_command($vmid, $cmd);
4861 }
4862
4863 sub vm_commandline {
4864     my ($storecfg, $vmid) = @_;
4865
4866     my $conf = PVE::QemuConfig->load_config($vmid);
4867
4868     my $defaults = load_defaults();
4869
4870     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4871
4872     return PVE::Tools::cmd2string($cmd);
4873 }
4874
4875 sub vm_reset {
4876     my ($vmid, $skiplock) = @_;
4877
4878     PVE::QemuConfig->lock_config($vmid, sub {
4879
4880         my $conf = PVE::QemuConfig->load_config($vmid);
4881
4882         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4883
4884         vm_mon_cmd($vmid, "system_reset");
4885     });
4886 }
4887
4888 sub get_vm_volumes {
4889     my ($conf) = @_;
4890
4891     my $vollist = [];
4892     foreach_volid($conf, sub {
4893         my ($volid, $attr) = @_;
4894
4895         return if $volid =~ m|^/|;
4896
4897         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4898         return if !$sid;
4899
4900         push @$vollist, $volid;
4901     });
4902
4903     return $vollist;
4904 }
4905
4906 sub vm_stop_cleanup {
4907     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4908
4909     eval {
4910
4911         if (!$keepActive) {
4912             my $vollist = get_vm_volumes($conf);
4913             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4914         }
4915
4916         foreach my $ext (qw(mon qmp pid vnc qga)) {
4917             unlink "/var/run/qemu-server/${vmid}.$ext";
4918         }
4919
4920         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4921     };
4922     warn $@ if $@; # avoid errors - just warn
4923 }
4924
4925 # Note: use $nockeck to skip tests if VM configuration file exists.
4926 # We need that when migration VMs to other nodes (files already moved)
4927 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4928 sub vm_stop {
4929     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4930
4931     $force = 1 if !defined($force) && !$shutdown;
4932
4933     if ($migratedfrom){
4934         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4935         kill 15, $pid if $pid;
4936         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4937         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4938         return;
4939     }
4940
4941     PVE::QemuConfig->lock_config($vmid, sub {
4942
4943         my $pid = check_running($vmid, $nocheck);
4944         return if !$pid;
4945
4946         my $conf;
4947         if (!$nocheck) {
4948             $conf = PVE::QemuConfig->load_config($vmid);
4949             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4950             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4951                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4952                 $timeout = $opts->{down} if $opts->{down};
4953             }
4954         }
4955
4956         $timeout = 60 if !defined($timeout);
4957
4958         eval {
4959             if ($shutdown) {
4960                 if (defined($conf) && $conf->{agent}) {
4961                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4962                 } else {
4963                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4964                 }
4965             } else {
4966                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4967             }
4968         };
4969         my $err = $@;
4970
4971         if (!$err) {
4972             my $count = 0;
4973             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4974                 $count++;
4975                 sleep 1;
4976             }
4977
4978             if ($count >= $timeout) {
4979                 if ($force) {
4980                     warn "VM still running - terminating now with SIGTERM\n";
4981                     kill 15, $pid;
4982                 } else {
4983                     die "VM quit/powerdown failed - got timeout\n";
4984                 }
4985             } else {
4986                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4987                 return;
4988             }
4989         } else {
4990             if ($force) {
4991                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4992                 kill 15, $pid;
4993             } else {
4994                 die "VM quit/powerdown failed\n";
4995             }
4996         }
4997
4998         # wait again
4999         $timeout = 10;
5000
5001         my $count = 0;
5002         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5003             $count++;
5004             sleep 1;
5005         }
5006
5007         if ($count >= $timeout) {
5008             warn "VM still running - terminating now with SIGKILL\n";
5009             kill 9, $pid;
5010             sleep 1;
5011         }
5012
5013         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5014    });
5015 }
5016
5017 sub vm_suspend {
5018     my ($vmid, $skiplock) = @_;
5019
5020     PVE::QemuConfig->lock_config($vmid, sub {
5021
5022         my $conf = PVE::QemuConfig->load_config($vmid);
5023
5024         PVE::QemuConfig->check_lock($conf)
5025             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5026
5027         vm_mon_cmd($vmid, "stop");
5028     });
5029 }
5030
5031 sub vm_resume {
5032     my ($vmid, $skiplock, $nocheck) = @_;
5033
5034     PVE::QemuConfig->lock_config($vmid, sub {
5035
5036         if (!$nocheck) {
5037
5038             my $conf = PVE::QemuConfig->load_config($vmid);
5039
5040             PVE::QemuConfig->check_lock($conf)
5041                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5042
5043             vm_mon_cmd($vmid, "cont");
5044
5045         } else {
5046             vm_mon_cmd_nocheck($vmid, "cont");
5047         }
5048     });
5049 }
5050
5051 sub vm_sendkey {
5052     my ($vmid, $skiplock, $key) = @_;
5053
5054     PVE::QemuConfig->lock_config($vmid, sub {
5055
5056         my $conf = PVE::QemuConfig->load_config($vmid);
5057
5058         # there is no qmp command, so we use the human monitor command
5059         vm_human_monitor_command($vmid, "sendkey $key");
5060     });
5061 }
5062
5063 sub vm_destroy {
5064     my ($storecfg, $vmid, $skiplock) = @_;
5065
5066     PVE::QemuConfig->lock_config($vmid, sub {
5067
5068         my $conf = PVE::QemuConfig->load_config($vmid);
5069
5070         if (!check_running($vmid)) {
5071             destroy_vm($storecfg, $vmid, undef, $skiplock);
5072         } else {
5073             die "VM $vmid is running - destroy failed\n";
5074         }
5075     });
5076 }
5077
5078 # pci helpers
5079
5080 sub file_write {
5081     my ($filename, $buf) = @_;
5082
5083     my $fh = IO::File->new($filename, "w");
5084     return undef if !$fh;
5085
5086     my $res = print $fh $buf;
5087
5088     $fh->close();
5089
5090     return $res;
5091 }
5092
5093 sub pci_device_info {
5094     my ($name) = @_;
5095
5096     my $res;
5097
5098     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5099     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5100
5101     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5102     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5103
5104     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5105     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5106
5107     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5108     return undef if !defined($product) || $product !~ s/^0x//;
5109
5110     $res = {
5111         name => $name,
5112         vendor => $vendor,
5113         product => $product,
5114         domain => $domain,
5115         bus => $bus,
5116         slot => $slot,
5117         func => $func,
5118         irq => $irq,
5119         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5120     };
5121
5122     return $res;
5123 }
5124
5125 sub pci_dev_reset {
5126     my ($dev) = @_;
5127
5128     my $name = $dev->{name};
5129
5130     my $fn = "$pcisysfs/devices/$name/reset";
5131
5132     return file_write($fn, "1");
5133 }
5134
5135 sub pci_dev_bind_to_vfio {
5136     my ($dev) = @_;
5137
5138     my $name = $dev->{name};
5139
5140     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5141
5142     if (!-d $vfio_basedir) {
5143         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5144     }
5145     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5146
5147     my $testdir = "$vfio_basedir/$name";
5148     return 1 if -d $testdir;
5149
5150     my $data = "$dev->{vendor} $dev->{product}";
5151     return undef if !file_write("$vfio_basedir/new_id", $data);
5152
5153     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5154     if (!file_write($fn, $name)) {
5155         return undef if -f $fn;
5156     }
5157
5158     $fn = "$vfio_basedir/bind";
5159     if (! -d $testdir) {
5160         return undef if !file_write($fn, $name);
5161     }
5162
5163     return -d $testdir;
5164 }
5165
5166 sub pci_dev_group_bind_to_vfio {
5167     my ($pciid) = @_;
5168
5169     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5170
5171     if (!-d $vfio_basedir) {
5172         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5173     }
5174     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5175
5176     # get IOMMU group devices
5177     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5178       my @devs = grep /^0000:/, readdir($D);
5179     closedir($D);
5180
5181     foreach my $pciid (@devs) {
5182         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5183
5184         # pci bridges, switches or root ports are not supported
5185         # they have a pci_bus subdirectory so skip them
5186         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5187
5188         my $info = pci_device_info($1);
5189         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5190     }
5191
5192     return 1;
5193 }
5194
5195 # vzdump restore implementaion
5196
5197 sub tar_archive_read_firstfile {
5198     my $archive = shift;
5199
5200     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5201
5202     # try to detect archive type first
5203     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5204         die "unable to open file '$archive'\n";
5205     my $firstfile = <$fh>;
5206     kill 15, $pid;
5207     close $fh;
5208
5209     die "ERROR: archive contaions no data\n" if !$firstfile;
5210     chomp $firstfile;
5211
5212     return $firstfile;
5213 }
5214
5215 sub tar_restore_cleanup {
5216     my ($storecfg, $statfile) = @_;
5217
5218     print STDERR "starting cleanup\n";
5219
5220     if (my $fd = IO::File->new($statfile, "r")) {
5221         while (defined(my $line = <$fd>)) {
5222             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5223                 my $volid = $2;
5224                 eval {
5225                     if ($volid =~ m|^/|) {
5226                         unlink $volid || die 'unlink failed\n';
5227                     } else {
5228                         PVE::Storage::vdisk_free($storecfg, $volid);
5229                     }
5230                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5231                 };
5232                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5233             } else {
5234                 print STDERR "unable to parse line in statfile - $line";
5235             }
5236         }
5237         $fd->close();
5238     }
5239 }
5240
5241 sub restore_archive {
5242     my ($archive, $vmid, $user, $opts) = @_;
5243
5244     my $format = $opts->{format};
5245     my $comp;
5246
5247     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5248         $format = 'tar' if !$format;
5249         $comp = 'gzip';
5250     } elsif ($archive =~ m/\.tar$/) {
5251         $format = 'tar' if !$format;
5252     } elsif ($archive =~ m/.tar.lzo$/) {
5253         $format = 'tar' if !$format;
5254         $comp = 'lzop';
5255     } elsif ($archive =~ m/\.vma$/) {
5256         $format = 'vma' if !$format;
5257     } elsif ($archive =~ m/\.vma\.gz$/) {
5258         $format = 'vma' if !$format;
5259         $comp = 'gzip';
5260     } elsif ($archive =~ m/\.vma\.lzo$/) {
5261         $format = 'vma' if !$format;
5262         $comp = 'lzop';
5263     } else {
5264         $format = 'vma' if !$format; # default
5265     }
5266
5267     # try to detect archive format
5268     if ($format eq 'tar') {
5269         return restore_tar_archive($archive, $vmid, $user, $opts);
5270     } else {
5271         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5272     }
5273 }
5274
5275 sub restore_update_config_line {
5276     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5277
5278     return if $line =~ m/^\#qmdump\#/;
5279     return if $line =~ m/^\#vzdump\#/;
5280     return if $line =~ m/^lock:/;
5281     return if $line =~ m/^unused\d+:/;
5282     return if $line =~ m/^parent:/;
5283     return if $line =~ m/^template:/; # restored VM is never a template
5284
5285     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5286     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5287         # try to convert old 1.X settings
5288         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5289         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5290             my ($model, $macaddr) = split(/\=/, $devconfig);
5291             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5292             my $net = {
5293                 model => $model,
5294                 bridge => "vmbr$ind",
5295                 macaddr => $macaddr,
5296             };
5297             my $netstr = print_net($net);
5298
5299             print $outfd "net$cookie->{netcount}: $netstr\n";
5300             $cookie->{netcount}++;
5301         }
5302     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5303         my ($id, $netstr) = ($1, $2);
5304         my $net = parse_net($netstr);
5305         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5306         $netstr = print_net($net);
5307         print $outfd "$id: $netstr\n";
5308     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5309         my $virtdev = $1;
5310         my $value = $3;
5311         my $di = parse_drive($virtdev, $value);
5312         if (defined($di->{backup}) && !$di->{backup}) {
5313             print $outfd "#$line";
5314         } elsif ($map->{$virtdev}) {
5315             delete $di->{format}; # format can change on restore
5316             $di->{file} = $map->{$virtdev};
5317             $value = print_drive($vmid, $di);
5318             print $outfd "$virtdev: $value\n";
5319         } else {
5320             print $outfd $line;
5321         }
5322     } else {
5323         print $outfd $line;
5324     }
5325 }
5326
5327 sub scan_volids {
5328     my ($cfg, $vmid) = @_;
5329
5330     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5331
5332     my $volid_hash = {};
5333     foreach my $storeid (keys %$info) {
5334         foreach my $item (@{$info->{$storeid}}) {
5335             next if !($item->{volid} && $item->{size});
5336             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5337             $volid_hash->{$item->{volid}} = $item;
5338         }
5339     }
5340
5341     return $volid_hash;
5342 }
5343
5344 sub is_volume_in_use {
5345     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5346
5347     my $path = PVE::Storage::path($storecfg, $volid);
5348
5349     my $scan_config = sub {
5350         my ($cref, $snapname) = @_;
5351
5352         foreach my $key (keys %$cref) {
5353             my $value = $cref->{$key};
5354             if (is_valid_drivename($key)) {
5355                 next if $skip_drive && $key eq $skip_drive;
5356                 my $drive = parse_drive($key, $value);
5357                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5358                 return 1 if $volid eq $drive->{file};
5359                 if ($drive->{file} =~ m!^/!) {
5360                     return 1 if $drive->{file} eq $path;
5361                 } else {
5362                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5363                     next if !$storeid;
5364                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5365                     next if !$scfg;
5366                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5367                 }
5368             }
5369         }
5370
5371         return 0;
5372     };
5373
5374     return 1 if &$scan_config($conf);
5375
5376     undef $skip_drive;
5377
5378     foreach my $snapname (keys %{$conf->{snapshots}}) {
5379         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5380     }
5381
5382     return 0;
5383 }
5384
5385 sub update_disksize {
5386     my ($vmid, $conf, $volid_hash) = @_;
5387
5388     my $changes;
5389
5390     # used and unused disks
5391     my $referenced = {};
5392
5393     # Note: it is allowed to define multiple storages with same path (alias), so
5394     # we need to check both 'volid' and real 'path' (two different volid can point
5395     # to the same path).
5396
5397     my $referencedpath = {};
5398
5399     # update size info
5400     foreach my $opt (keys %$conf) {
5401         if (is_valid_drivename($opt)) {
5402             my $drive = parse_drive($opt, $conf->{$opt});
5403             my $volid = $drive->{file};
5404             next if !$volid;
5405
5406             $referenced->{$volid} = 1;
5407             if ($volid_hash->{$volid} &&
5408                 (my $path = $volid_hash->{$volid}->{path})) {
5409                 $referencedpath->{$path} = 1;
5410             }
5411
5412             next if drive_is_cdrom($drive);
5413             next if !$volid_hash->{$volid};
5414
5415             $drive->{size} = $volid_hash->{$volid}->{size};
5416             my $new = print_drive($vmid, $drive);
5417             if ($new ne $conf->{$opt}) {
5418                 $changes = 1;
5419                 $conf->{$opt} = $new;
5420             }
5421         }
5422     }
5423
5424     # remove 'unusedX' entry if volume is used
5425     foreach my $opt (keys %$conf) {
5426         next if $opt !~ m/^unused\d+$/;
5427         my $volid = $conf->{$opt};
5428         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5429         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5430             $changes = 1;
5431             delete $conf->{$opt};
5432         }
5433
5434         $referenced->{$volid} = 1;
5435         $referencedpath->{$path} = 1 if $path;
5436     }
5437
5438     foreach my $volid (sort keys %$volid_hash) {
5439         next if $volid =~ m/vm-$vmid-state-/;
5440         next if $referenced->{$volid};
5441         my $path = $volid_hash->{$volid}->{path};
5442         next if !$path; # just to be sure
5443         next if $referencedpath->{$path};
5444         $changes = 1;
5445         PVE::QemuConfig->add_unused_volume($conf, $volid);
5446         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5447     }
5448
5449     return $changes;
5450 }
5451
5452 sub rescan {
5453     my ($vmid, $nolock) = @_;
5454
5455     my $cfg = PVE::Storage::config();
5456
5457     my $volid_hash = scan_volids($cfg, $vmid);
5458
5459     my $updatefn =  sub {
5460         my ($vmid) = @_;
5461
5462         my $conf = PVE::QemuConfig->load_config($vmid);
5463
5464         PVE::QemuConfig->check_lock($conf);
5465
5466         my $vm_volids = {};
5467         foreach my $volid (keys %$volid_hash) {
5468             my $info = $volid_hash->{$volid};
5469             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5470         }
5471
5472         my $changes = update_disksize($vmid, $conf, $vm_volids);
5473
5474         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5475     };
5476
5477     if (defined($vmid)) {
5478         if ($nolock) {
5479             &$updatefn($vmid);
5480         } else {
5481             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5482         }
5483     } else {
5484         my $vmlist = config_list();
5485         foreach my $vmid (keys %$vmlist) {
5486             if ($nolock) {
5487                 &$updatefn($vmid);
5488             } else {
5489                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5490             }
5491         }
5492     }
5493 }
5494
5495 sub restore_vma_archive {
5496     my ($archive, $vmid, $user, $opts, $comp) = @_;
5497
5498     my $input = $archive eq '-' ? "<&STDIN" : undef;
5499     my $readfrom = $archive;
5500
5501     my $uncomp = '';
5502     if ($comp) {
5503         $readfrom = '-';
5504         my $qarchive = PVE::Tools::shellquote($archive);
5505         if ($comp eq 'gzip') {
5506             $uncomp = "zcat $qarchive|";
5507         } elsif ($comp eq 'lzop') {
5508             $uncomp = "lzop -d -c $qarchive|";
5509         } else {
5510             die "unknown compression method '$comp'\n";
5511         }
5512
5513     }
5514
5515     my $tmpdir = "/var/tmp/vzdumptmp$$";
5516     rmtree $tmpdir;
5517
5518     # disable interrupts (always do cleanups)
5519     local $SIG{INT} =
5520         local $SIG{TERM} =
5521         local $SIG{QUIT} =
5522         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5523
5524     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5525     POSIX::mkfifo($mapfifo, 0600);
5526     my $fifofh;
5527
5528     my $openfifo = sub {
5529         open($fifofh, '>', $mapfifo) || die $!;
5530     };
5531
5532     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5533
5534     my $oldtimeout;
5535     my $timeout = 5;
5536
5537     my $devinfo = {};
5538
5539     my $rpcenv = PVE::RPCEnvironment::get();
5540
5541     my $conffile = PVE::QemuConfig->config_file($vmid);
5542     my $tmpfn = "$conffile.$$.tmp";
5543
5544     # Note: $oldconf is undef if VM does not exists
5545     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5546     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5547
5548     my $print_devmap = sub {
5549         my $virtdev_hash = {};
5550
5551         my $cfgfn = "$tmpdir/qemu-server.conf";
5552
5553         # we can read the config - that is already extracted
5554         my $fh = IO::File->new($cfgfn, "r") ||
5555             "unable to read qemu-server.conf - $!\n";
5556
5557         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5558         if (-f $fwcfgfn) {
5559             my $pve_firewall_dir = '/etc/pve/firewall';
5560             mkdir $pve_firewall_dir; # make sure the dir exists
5561             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5562         }
5563
5564         while (defined(my $line = <$fh>)) {
5565             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5566                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5567                 die "archive does not contain data for drive '$virtdev'\n"
5568                     if !$devinfo->{$devname};
5569                 if (defined($opts->{storage})) {
5570                     $storeid = $opts->{storage} || 'local';
5571                 } elsif (!$storeid) {
5572                     $storeid = 'local';
5573                 }
5574                 $format = 'raw' if !$format;
5575                 $devinfo->{$devname}->{devname} = $devname;
5576                 $devinfo->{$devname}->{virtdev} = $virtdev;
5577                 $devinfo->{$devname}->{format} = $format;
5578                 $devinfo->{$devname}->{storeid} = $storeid;
5579
5580                 # check permission on storage
5581                 my $pool = $opts->{pool}; # todo: do we need that?
5582                 if ($user ne 'root@pam') {
5583                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5584                 }
5585
5586                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5587             }
5588         }
5589
5590         foreach my $devname (keys %$devinfo) {
5591             die "found no device mapping information for device '$devname'\n"
5592                 if !$devinfo->{$devname}->{virtdev};
5593         }
5594
5595         my $cfg = PVE::Storage::config();
5596
5597         # create empty/temp config
5598         if ($oldconf) {
5599             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5600             foreach_drive($oldconf, sub {
5601                 my ($ds, $drive) = @_;
5602
5603                 return if drive_is_cdrom($drive);
5604
5605                 my $volid = $drive->{file};
5606
5607                 return if !$volid || $volid =~ m|^/|;
5608
5609                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5610                 return if !$path || !$owner || ($owner != $vmid);
5611
5612                 # Note: only delete disk we want to restore
5613                 # other volumes will become unused
5614                 if ($virtdev_hash->{$ds}) {
5615                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5616                     if (my $err = $@) {
5617                         warn $err;
5618                     }
5619                 }
5620             });
5621
5622             # delete vmstate files
5623             # since after the restore we have no snapshots anymore
5624             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5625                 my $snap = $oldconf->{snapshots}->{$snapname};
5626                 if ($snap->{vmstate}) {
5627                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5628                     if (my $err = $@) {
5629                         warn $err;
5630                     }
5631                 }
5632             }
5633         }
5634
5635         my $map = {};
5636         foreach my $virtdev (sort keys %$virtdev_hash) {
5637             my $d = $virtdev_hash->{$virtdev};
5638             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5639             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5640
5641             # test if requested format is supported
5642             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5643             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5644             $d->{format} = $defFormat if !$supported;
5645
5646             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5647                                                   $d->{format}, undef, $alloc_size);
5648             print STDERR "new volume ID is '$volid'\n";
5649             $d->{volid} = $volid;
5650             my $path = PVE::Storage::path($cfg, $volid);
5651
5652             PVE::Storage::activate_volumes($cfg,[$volid]);
5653
5654             my $write_zeros = 1;
5655             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5656                 $write_zeros = 0;
5657             }
5658
5659             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5660
5661             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5662             $map->{$virtdev} = $volid;
5663         }
5664
5665         $fh->seek(0, 0) || die "seek failed - $!\n";
5666
5667         my $outfd = new IO::File ($tmpfn, "w") ||
5668             die "unable to write config for VM $vmid\n";
5669
5670         my $cookie = { netcount => 0 };
5671         while (defined(my $line = <$fh>)) {
5672             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5673         }
5674
5675         $fh->close();
5676         $outfd->close();
5677     };
5678
5679     eval {
5680         # enable interrupts
5681         local $SIG{INT} =
5682             local $SIG{TERM} =
5683             local $SIG{QUIT} =
5684             local $SIG{HUP} =
5685             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5686         local $SIG{ALRM} = sub { die "got timeout\n"; };
5687
5688         $oldtimeout = alarm($timeout);
5689
5690         my $parser = sub {
5691             my $line = shift;
5692
5693             print "$line\n";
5694
5695             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5696                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5697                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5698             } elsif ($line =~ m/^CTIME: /) {
5699                 # we correctly received the vma config, so we can disable
5700                 # the timeout now for disk allocation (set to 10 minutes, so
5701                 # that we always timeout if something goes wrong)
5702                 alarm(600);
5703                 &$print_devmap();
5704                 print $fifofh "done\n";
5705                 my $tmp = $oldtimeout || 0;
5706                 $oldtimeout = undef;
5707                 alarm($tmp);
5708                 close($fifofh);
5709             }
5710         };
5711
5712         print "restore vma archive: $cmd\n";
5713         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5714     };
5715     my $err = $@;
5716
5717     alarm($oldtimeout) if $oldtimeout;
5718
5719     my $vollist = [];
5720     foreach my $devname (keys %$devinfo) {
5721         my $volid = $devinfo->{$devname}->{volid};
5722         push @$vollist, $volid if $volid;
5723     }
5724
5725     my $cfg = PVE::Storage::config();
5726     PVE::Storage::deactivate_volumes($cfg, $vollist);
5727
5728     unlink $mapfifo;
5729
5730     if ($err) {
5731         rmtree $tmpdir;
5732         unlink $tmpfn;
5733
5734         foreach my $devname (keys %$devinfo) {
5735             my $volid = $devinfo->{$devname}->{volid};
5736             next if !$volid;
5737             eval {
5738                 if ($volid =~ m|^/|) {
5739                     unlink $volid || die 'unlink failed\n';
5740                 } else {
5741                     PVE::Storage::vdisk_free($cfg, $volid);
5742                 }
5743                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5744             };
5745             print STDERR "unable to cleanup '$volid' - $@" if $@;
5746         }
5747         die $err;
5748     }
5749
5750     rmtree $tmpdir;
5751
5752     rename($tmpfn, $conffile) ||
5753         die "unable to commit configuration file '$conffile'\n";
5754
5755     PVE::Cluster::cfs_update(); # make sure we read new file
5756
5757     eval { rescan($vmid, 1); };
5758     warn $@ if $@;
5759 }
5760
5761 sub restore_tar_archive {
5762     my ($archive, $vmid, $user, $opts) = @_;
5763
5764     if ($archive ne '-') {
5765         my $firstfile = tar_archive_read_firstfile($archive);
5766         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5767             if $firstfile ne 'qemu-server.conf';
5768     }
5769
5770     my $storecfg = PVE::Storage::config();
5771
5772     # destroy existing data - keep empty config
5773     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5774     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5775
5776     my $tocmd = "/usr/lib/qemu-server/qmextract";
5777
5778     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5779     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5780     $tocmd .= ' --prealloc' if $opts->{prealloc};
5781     $tocmd .= ' --info' if $opts->{info};
5782
5783     # tar option "xf" does not autodetect compression when read from STDIN,
5784     # so we pipe to zcat
5785     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5786         PVE::Tools::shellquote("--to-command=$tocmd");
5787
5788     my $tmpdir = "/var/tmp/vzdumptmp$$";
5789     mkpath $tmpdir;
5790
5791     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5792     local $ENV{VZDUMP_VMID} = $vmid;
5793     local $ENV{VZDUMP_USER} = $user;
5794
5795     my $conffile = PVE::QemuConfig->config_file($vmid);
5796     my $tmpfn = "$conffile.$$.tmp";
5797
5798     # disable interrupts (always do cleanups)
5799     local $SIG{INT} =
5800         local $SIG{TERM} =
5801         local $SIG{QUIT} =
5802         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5803
5804     eval {
5805         # enable interrupts
5806         local $SIG{INT} =
5807             local $SIG{TERM} =
5808             local $SIG{QUIT} =
5809             local $SIG{HUP} =
5810             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5811
5812         if ($archive eq '-') {
5813             print "extracting archive from STDIN\n";
5814             run_command($cmd, input => "<&STDIN");
5815         } else {
5816             print "extracting archive '$archive'\n";
5817             run_command($cmd);
5818         }
5819
5820         return if $opts->{info};
5821
5822         # read new mapping
5823         my $map = {};
5824         my $statfile = "$tmpdir/qmrestore.stat";
5825         if (my $fd = IO::File->new($statfile, "r")) {
5826             while (defined (my $line = <$fd>)) {
5827                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5828                     $map->{$1} = $2 if $1;
5829                 } else {
5830                     print STDERR "unable to parse line in statfile - $line\n";
5831                 }
5832             }
5833             $fd->close();
5834         }
5835
5836         my $confsrc = "$tmpdir/qemu-server.conf";
5837
5838         my $srcfd = new IO::File($confsrc, "r") ||
5839             die "unable to open file '$confsrc'\n";
5840
5841         my $outfd = new IO::File ($tmpfn, "w") ||
5842             die "unable to write config for VM $vmid\n";
5843
5844         my $cookie = { netcount => 0 };
5845         while (defined (my $line = <$srcfd>)) {
5846             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5847         }
5848
5849         $srcfd->close();
5850         $outfd->close();
5851     };
5852     my $err = $@;
5853
5854     if ($err) {
5855
5856         unlink $tmpfn;
5857
5858         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5859
5860         die $err;
5861     }
5862
5863     rmtree $tmpdir;
5864
5865     rename $tmpfn, $conffile ||
5866         die "unable to commit configuration file '$conffile'\n";
5867
5868     PVE::Cluster::cfs_update(); # make sure we read new file
5869
5870     eval { rescan($vmid, 1); };
5871     warn $@ if $@;
5872 };
5873
5874 sub foreach_storage_used_by_vm {
5875     my ($conf, $func) = @_;
5876
5877     my $sidhash = {};
5878
5879     foreach_drive($conf, sub {
5880         my ($ds, $drive) = @_;
5881         return if drive_is_cdrom($drive);
5882
5883         my $volid = $drive->{file};
5884
5885         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5886         $sidhash->{$sid} = $sid if $sid;
5887     });
5888
5889     foreach my $sid (sort keys %$sidhash) {
5890         &$func($sid);
5891     }
5892 }
5893
5894 sub do_snapshots_with_qemu {
5895     my ($storecfg, $volid) = @_;
5896
5897     my $storage_name = PVE::Storage::parse_volume_id($volid);
5898
5899     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5900         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5901         return 1;
5902     }
5903
5904     if ($volid =~ m/\.(qcow2|qed)$/){
5905         return 1;
5906     }
5907
5908     return undef;
5909 }
5910
5911 sub qga_check_running {
5912     my ($vmid) = @_;
5913
5914     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5915     if ($@) {
5916         warn "Qemu Guest Agent is not running - $@";
5917         return 0;
5918     }
5919     return 1;
5920 }
5921
5922 sub template_create {
5923     my ($vmid, $conf, $disk) = @_;
5924
5925     my $storecfg = PVE::Storage::config();
5926
5927     foreach_drive($conf, sub {
5928         my ($ds, $drive) = @_;
5929
5930         return if drive_is_cdrom($drive);
5931         return if $disk && $ds ne $disk;
5932
5933         my $volid = $drive->{file};
5934         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5935
5936         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5937         $drive->{file} = $voliddst;
5938         $conf->{$ds} = print_drive($vmid, $drive);
5939         PVE::QemuConfig->write_config($vmid, $conf);
5940     });
5941 }
5942
5943 sub qemu_img_convert {
5944     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5945
5946     my $storecfg = PVE::Storage::config();
5947     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5948     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5949
5950     if ($src_storeid && $dst_storeid) {
5951
5952         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5953
5954         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5955         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5956
5957         my $src_format = qemu_img_format($src_scfg, $src_volname);
5958         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5959
5960         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5961         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5962
5963         my $cmd = [];
5964         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5965         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5966         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5967         if ($is_zero_initialized) {
5968             push @$cmd, "zeroinit:$dst_path";
5969         } else {
5970             push @$cmd, $dst_path;
5971         }
5972
5973         my $parser = sub {
5974             my $line = shift;
5975             if($line =~ m/\((\S+)\/100\%\)/){
5976                 my $percent = $1;
5977                 my $transferred = int($size * $percent / 100);
5978                 my $remaining = $size - $transferred;
5979
5980                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5981             }
5982
5983         };
5984
5985         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5986         my $err = $@;
5987         die "copy failed: $err" if $err;
5988     }
5989 }
5990
5991 sub qemu_img_format {
5992     my ($scfg, $volname) = @_;
5993
5994     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5995         return $1;
5996     } else {
5997         return "raw";
5998     }
5999 }
6000
6001 sub qemu_drive_mirror {
6002     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6003
6004     $jobs = {} if !$jobs;
6005
6006     my $qemu_target;
6007     my $format;
6008     $jobs->{"drive-$drive"} = {};
6009
6010     if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
6011         my $server = $1;
6012         my $port = $2;
6013         my $exportname = $3;
6014
6015         $format = "nbd";
6016         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
6017         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
6018         my $cmd = ['socat', '-T30', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=5"];
6019
6020         my $pid = fork();
6021         if (!defined($pid)) {
6022             die "forking socat tunnel failed\n";
6023         } elsif ($pid == 0) {
6024             exec(@$cmd);
6025             warn "exec failed: $!\n";
6026             POSIX::_exit(-1);
6027         }
6028         $jobs->{"drive-$drive"}->{pid} = $pid;
6029
6030         my $timeout = 0;
6031         while (!-S $unixsocket) {
6032             die "nbd connection helper timed out\n"
6033                 if $timeout++ > 5;
6034             sleep 1;
6035         }
6036     } else {
6037         my $storecfg = PVE::Storage::config();
6038         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6039
6040         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6041
6042         $format = qemu_img_format($dst_scfg, $dst_volname);
6043
6044         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6045
6046         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6047     }
6048
6049     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6050     $opts->{format} = $format if $format;
6051
6052     print "drive mirror is starting for drive-$drive\n";
6053
6054     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6055
6056     if (my $err = $@) {
6057         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6058         die "mirroring error: $err";
6059     }
6060
6061     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6062 }
6063
6064 sub qemu_drive_mirror_monitor {
6065     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6066
6067     eval {
6068         my $err_complete = 0;
6069
6070         while (1) {
6071             die "storage migration timed out\n" if $err_complete > 300;
6072
6073             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6074
6075             my $running_mirror_jobs = {};
6076             foreach my $stat (@$stats) {
6077                 next if $stat->{type} ne 'mirror';
6078                 $running_mirror_jobs->{$stat->{device}} = $stat;
6079             }
6080
6081             my $readycounter = 0;
6082
6083             foreach my $job (keys %$jobs) {
6084
6085                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6086                     print "$job : finished\n";
6087                     delete $jobs->{$job};
6088                     next;
6089                 }
6090
6091                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6092
6093                 my $busy = $running_mirror_jobs->{$job}->{busy};
6094                 my $ready = $running_mirror_jobs->{$job}->{ready};
6095                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6096                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6097                     my $remaining = $total - $transferred;
6098                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6099
6100                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6101                 }
6102
6103                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6104             }
6105
6106             last if scalar(keys %$jobs) == 0;
6107
6108             if ($readycounter == scalar(keys %$jobs)) {
6109                 print "all mirroring jobs are ready \n";
6110                 last if $skipcomplete; #do the complete later
6111
6112                 if ($vmiddst && $vmiddst != $vmid) {
6113                     my $agent_running = $qga && qga_check_running($vmid);
6114                     if ($agent_running) {
6115                         print "freeze filesystem\n";
6116                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6117                     } else {
6118                         print "suspend vm\n";
6119                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6120                     }
6121
6122                     # if we clone a disk for a new target vm, we don't switch the disk
6123                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6124
6125                     if ($agent_running) {
6126                         print "unfreeze filesystem\n";
6127                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6128                     } else {
6129                         print "resume vm\n";
6130                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6131                     }
6132
6133                     last;
6134                 } else {
6135
6136                     foreach my $job (keys %$jobs) {
6137                         # try to switch the disk if source and destination are on the same guest
6138                         print "$job: Completing block job...\n";
6139
6140                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6141                         if ($@ =~ m/cannot be completed/) {
6142                             print "$job: Block job cannot be completed, try again.\n";
6143                             $err_complete++;
6144                         }else {
6145                             print "$job: Completed successfully.\n";
6146                             $jobs->{$job}->{complete} = 1;
6147                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6148                         }
6149                     }
6150                 }
6151             }
6152             sleep 1;
6153         }
6154     };
6155     my $err = $@;
6156
6157     if ($err) {
6158         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6159         die "mirroring error: $err";
6160     }
6161
6162 }
6163
6164 sub qemu_blockjobs_cancel {
6165     my ($vmid, $jobs) = @_;
6166
6167     foreach my $job (keys %$jobs) {
6168         print "$job: Cancelling block job\n";
6169         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6170         $jobs->{$job}->{cancel} = 1;
6171     }
6172
6173     while (1) {
6174         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6175
6176         my $running_jobs = {};
6177         foreach my $stat (@$stats) {
6178             $running_jobs->{$stat->{device}} = $stat;
6179         }
6180
6181         foreach my $job (keys %$jobs) {
6182
6183             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6184                 print "$job: Done.\n";
6185                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6186                 delete $jobs->{$job};
6187             }
6188         }
6189
6190         last if scalar(keys %$jobs) == 0;
6191
6192         sleep 1;
6193     }
6194 }
6195
6196 sub qemu_blockjobs_finish_tunnel {
6197    my ($vmid, $job, $cpid) = @_;
6198
6199    return if !$cpid;
6200
6201    for (my $i = 1; $i < 20; $i++) {
6202         my $waitpid = waitpid($cpid, WNOHANG);
6203         last if (defined($waitpid) && ($waitpid == $cpid));
6204
6205         if ($i == 10) {
6206             kill(15, $cpid);
6207         } elsif ($i >= 15) {
6208             kill(9, $cpid);
6209         }
6210         sleep (1);
6211     }
6212     unlink "/run/qemu-server/$vmid.mirror-$job";
6213 }
6214
6215 sub clone_disk {
6216     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6217         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6218
6219     my $newvolid;
6220
6221     if (!$full) {
6222         print "create linked clone of drive $drivename ($drive->{file})\n";
6223         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6224         push @$newvollist, $newvolid;
6225     } else {
6226
6227         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6228         $storeid = $storage if $storage;
6229
6230         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6231         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6232
6233         print "create full clone of drive $drivename ($drive->{file})\n";
6234         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, undef, ($size/1024));
6235         push @$newvollist, $newvolid;
6236
6237         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6238
6239         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6240         if (!$running || $snapname) {
6241             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6242         } else {
6243
6244             my $kvmver = get_running_qemu_version ($vmid);
6245             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6246                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6247                     if $drive->{iothread};
6248             }
6249
6250             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6251         }
6252     }
6253
6254     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6255
6256     my $disk = $drive;
6257     $disk->{format} = undef;
6258     $disk->{file} = $newvolid;
6259     $disk->{size} = $size;
6260
6261     return $disk;
6262 }
6263
6264 # this only works if VM is running
6265 sub get_current_qemu_machine {
6266     my ($vmid) = @_;
6267
6268     my $cmd = { execute => 'query-machines', arguments => {} };
6269     my $res = vm_qmp_command($vmid, $cmd);
6270
6271     my ($current, $default);
6272     foreach my $e (@$res) {
6273         $default = $e->{name} if $e->{'is-default'};
6274         $current = $e->{name} if $e->{'is-current'};
6275     }
6276
6277     # fallback to the default machine if current is not supported by qemu
6278     return $current || $default || 'pc';
6279 }
6280
6281 sub get_running_qemu_version {
6282     my ($vmid) = @_;
6283     my $cmd = { execute => 'query-version', arguments => {} };
6284     my $res = vm_qmp_command($vmid, $cmd);
6285     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6286 }
6287
6288 sub qemu_machine_feature_enabled {
6289     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6290
6291     my $current_major;
6292     my $current_minor;
6293
6294     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6295
6296         $current_major = $3;
6297         $current_minor = $4;
6298
6299     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6300
6301         $current_major = $1;
6302         $current_minor = $2;
6303     }
6304
6305     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6306
6307
6308 }
6309
6310 sub qemu_machine_pxe {
6311     my ($vmid, $conf, $machine) = @_;
6312
6313     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6314
6315     foreach my $opt (keys %$conf) {
6316         next if $opt !~ m/^net(\d+)$/;
6317         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6318         next if !$net;
6319         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6320         return $machine.".pxe" if $romfile =~ m/pxe/;
6321         last;
6322     }
6323
6324     return $machine;
6325 }
6326
6327 sub qemu_use_old_bios_files {
6328     my ($machine_type) = @_;
6329
6330     return if !$machine_type;
6331
6332     my $use_old_bios_files = undef;
6333
6334     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6335         $machine_type = $1;
6336         $use_old_bios_files = 1;
6337     } else {
6338         my $kvmver = kvm_user_version();
6339         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6340         # load new efi bios files on migration. So this hack is required to allow
6341         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6342         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6343         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6344     }
6345
6346     return ($use_old_bios_files, $machine_type);
6347 }
6348
6349 sub create_efidisk {
6350     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6351
6352     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6353
6354     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6355     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6356     PVE::Storage::activate_volumes($storecfg, [$volid]);
6357
6358     my $path = PVE::Storage::path($storecfg, $volid);
6359     eval {
6360         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6361     };
6362     die "Copying EFI vars image failed: $@" if $@;
6363
6364     return ($volid, $vars_size);
6365 }
6366
6367 sub lspci {
6368
6369     my $devices = {};
6370
6371     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6372             my (undef, $id, $function) = @_;
6373             my $res = { id => $id, function => $function};
6374             push @{$devices->{$id}}, $res;
6375     });
6376
6377     # Entries should be sorted by functions.
6378     foreach my $id (keys %$devices) {
6379         my $dev = $devices->{$id};
6380         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6381     }
6382
6383     return $devices;
6384 }
6385
6386 sub vm_iothreads_list {
6387     my ($vmid) = @_;
6388
6389     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6390
6391     my $iothreads = {};
6392     foreach my $iothread (@$res) {
6393         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6394     }
6395
6396     return $iothreads;
6397 }
6398
6399 sub scsihw_infos {
6400     my ($conf, $drive) = @_;
6401
6402     my $maxdev = 0;
6403
6404     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6405         $maxdev = 7;
6406     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6407         $maxdev = 1;
6408     } else {
6409         $maxdev = 256;
6410     }
6411
6412     my $controller = int($drive->{index} / $maxdev);
6413     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6414
6415     return ($maxdev, $controller, $controller_prefix);
6416 }
6417
6418 sub add_hyperv_enlightenments {
6419     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6420
6421     return if $winversion < 6;
6422     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6423
6424     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6425
6426     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6427         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6428         push @$cpuFlags , 'hv_vapic';
6429         push @$cpuFlags , 'hv_time';
6430     } else {
6431         push @$cpuFlags , 'hv_spinlocks=0xffff';
6432     }
6433
6434     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6435         push @$cpuFlags , 'hv_reset';
6436         push @$cpuFlags , 'hv_vpindex';
6437         push @$cpuFlags , 'hv_runtime';
6438     }
6439
6440     if ($winversion >= 7) {
6441         push @$cpuFlags , 'hv_relaxed';
6442     }
6443 }
6444
6445 sub windows_version {
6446     my ($ostype) = @_;
6447
6448     return 0 if !$ostype;
6449
6450     my $winversion = 0;
6451
6452     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6453         $winversion = 5;
6454     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6455         $winversion = 6;
6456     } elsif ($ostype =~ m/^win(\d+)$/) {
6457         $winversion = $1;
6458     }
6459
6460     return $winversion;
6461 }
6462
6463 sub resolve_dst_disk_format {
6464         my ($storecfg, $storeid, $src_volname, $format) = @_;
6465         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6466
6467         if (!$format) {
6468             # if no target format is specified, use the source disk format as hint
6469             if ($src_volname) {
6470                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6471                 $format = qemu_img_format($scfg, $src_volname);
6472             } else {
6473                 return $defFormat;
6474             }
6475         }
6476
6477         # test if requested format is supported - else use default
6478         my $supported = grep { $_ eq $format } @$validFormats;
6479         $format = $defFormat if !$supported;
6480         return $format;
6481 }
6482
6483 sub resolve_first_disk {
6484     my $conf = shift;
6485     my @disks = PVE::QemuServer::valid_drive_names();
6486     my $firstdisk;
6487     foreach my $ds (reverse @disks) {
6488         next if !$conf->{$ds};
6489         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6490         next if PVE::QemuServer::drive_is_cdrom($disk);
6491         $firstdisk = $ds;
6492     }
6493     return $firstdisk;
6494 }
6495
6496 sub generate_smbios1_uuid {
6497     my ($uuid, $uuid_str);
6498     UUID::generate($uuid);
6499     UUID::unparse($uuid, $uuid_str);
6500     return "uuid=$uuid_str";
6501 }
6502
6503 # bash completion helper
6504
6505 sub complete_backup_archives {
6506     my ($cmdname, $pname, $cvalue) = @_;
6507
6508     my $cfg = PVE::Storage::config();
6509
6510     my $storeid;
6511
6512     if ($cvalue =~ m/^([^:]+):/) {
6513         $storeid = $1;
6514     }
6515
6516     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6517
6518     my $res = [];
6519     foreach my $id (keys %$data) {
6520         foreach my $item (@{$data->{$id}}) {
6521             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6522             push @$res, $item->{volid} if defined($item->{volid});
6523         }
6524     }
6525
6526     return $res;
6527 }
6528
6529 my $complete_vmid_full = sub {
6530     my ($running) = @_;
6531
6532     my $idlist = vmstatus();
6533
6534     my $res = [];
6535
6536     foreach my $id (keys %$idlist) {
6537         my $d = $idlist->{$id};
6538         if (defined($running)) {
6539             next if $d->{template};
6540             next if $running && $d->{status} ne 'running';
6541             next if !$running && $d->{status} eq 'running';
6542         }
6543         push @$res, $id;
6544
6545     }
6546     return $res;
6547 };
6548
6549 sub complete_vmid {
6550     return &$complete_vmid_full();
6551 }
6552
6553 sub complete_vmid_stopped {
6554     return &$complete_vmid_full(0);
6555 }
6556
6557 sub complete_vmid_running {
6558     return &$complete_vmid_full(1);
6559 }
6560
6561 sub complete_storage {
6562
6563     my $cfg = PVE::Storage::config();
6564     my $ids = $cfg->{ids};
6565
6566     my $res = [];
6567     foreach my $sid (keys %$ids) {
6568         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6569         next if !$ids->{$sid}->{content}->{images};
6570         push @$res, $sid;
6571     }
6572
6573     return $res;
6574 }
6575
6576 sub nbd_stop {
6577     my ($vmid) = @_;
6578
6579     vm_mon_cmd($vmid, 'nbd-server-stop');
6580 }
6581
6582 1;