PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42
  43 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  44
  45 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  46
  47 # Note about locking: we use flock on the config file protect
  48 # against concurent actions.
  49 # Aditionaly, we have a 'lock' setting in the config file. This
  50 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  51 # allowed when such lock is set. But you can ignore this kind of
  52 # lock with the --skiplock flag.
  53
  54 cfs_register_file('/qemu-server/',
  55                   \&parse_vm_config,
  56                   \&write_vm_config);
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  72     type => 'string',
  73     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  74     description => "The drive's backing file's data format.",
  75     optional => 1,
  76 });
  77
  78 #no warnings 'redefine';
  79
  80 sub cgroups_write {
  81    my ($controller, $vmid, $option, $value) = @_;
  82
  83    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  84    PVE::ProcFSTools::write_proc_entry($path, $value);
  85
  86 }
  87
  88 my $nodename = PVE::INotify::nodename();
  89
  90 mkdir "/etc/pve/nodes/$nodename";
  91 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  92 mkdir $confdir;
  93
  94 my $var_run_tmpdir = "/var/run/qemu-server";
  95 mkdir $var_run_tmpdir;
  96
  97 my $lock_dir = "/var/lock/qemu-server";
  98 mkdir $lock_dir;
  99
 100 my $pcisysfs = "/sys/bus/pci";
 101
 102 my $cpu_vendor_list = {
 103     # Intel CPUs
 104     486 => 'GenuineIntel',
 105     pentium => 'GenuineIntel',
 106     pentium2  => 'GenuineIntel',
 107     pentium3  => 'GenuineIntel',
 108     coreduo => 'GenuineIntel',
 109     core2duo => 'GenuineIntel',
 110     Conroe  => 'GenuineIntel',
 111     Penryn  => 'GenuineIntel',
 112     Nehalem  => 'GenuineIntel',
 113     Westmere => 'GenuineIntel',
 114     SandyBridge => 'GenuineIntel',
 115     IvyBridge => 'GenuineIntel',
 116     Haswell => 'GenuineIntel',
 117     'Haswell-noTSX' => 'GenuineIntel',
 118     Broadwell => 'GenuineIntel',
 119     'Broadwell-noTSX' => 'GenuineIntel',
 120     'Skylake-Client' => 'GenuineIntel',
 121
 122     # AMD CPUs
 123     athlon => 'AuthenticAMD',
 124     phenom  => 'AuthenticAMD',
 125     Opteron_G1  => 'AuthenticAMD',
 126     Opteron_G2  => 'AuthenticAMD',
 127     Opteron_G3  => 'AuthenticAMD',
 128     Opteron_G4  => 'AuthenticAMD',
 129     Opteron_G5  => 'AuthenticAMD',
 130
 131     # generic types, use vendor from host node
 132     host => 'default',
 133     kvm32 => 'default',
 134     kvm64 => 'default',
 135     qemu32 => 'default',
 136     qemu64 => 'default',
 137 };
 138
 139 my $cpu_fmt = {
 140     cputype => {
 141         description => "Emulated CPU type.",
 142         type => 'string',
 143         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 144         default => 'kvm64',
 145         default_key => 1,
 146     },
 147     hidden => {
 148         description => "Do not identify as a KVM virtual machine.",
 149         type => 'boolean',
 150         optional => 1,
 151         default => 0
 152     },
 153     flags => {
 154         description => "Override CPU flags. Currently only the 'pcid' flag is supported."
 155                      . " Use '+pcid' or '-pcid' to enable or disable."
 156                      . " This takes precedence over flags coming from the cpu type or changed implicitly via the OS type.",
 157         format_description => 'flaglist',
 158         type => 'string',
 159         pattern => '[+-]pcid',
 160         optional => 1,
 161         default => '',
 162     },
 163 };
 164
 165 my $watchdog_fmt = {
 166     model => {
 167         default_key => 1,
 168         type => 'string',
 169         enum => [qw(i6300esb ib700)],
 170         description => "Watchdog type to emulate.",
 171         default => 'i6300esb',
 172         optional => 1,
 173     },
 174     action => {
 175         type => 'string',
 176         enum => [qw(reset shutdown poweroff pause debug none)],
 177         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 178         optional => 1,
 179     },
 180 };
 181 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 182
 183 my $confdesc = {
 184     onboot => {
 185         optional => 1,
 186         type => 'boolean',
 187         description => "Specifies whether a VM will be started during system bootup.",
 188         default => 0,
 189     },
 190     autostart => {
 191         optional => 1,
 192         type => 'boolean',
 193         description => "Automatic restart after crash (currently ignored).",
 194         default => 0,
 195     },
 196     hotplug => {
 197         optional => 1,
 198         type => 'string', format => 'pve-hotplug-features',
 199         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 200         default => 'network,disk,usb',
 201     },
 202     reboot => {
 203         optional => 1,
 204         type => 'boolean',
 205         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 206         default => 1,
 207     },
 208     lock => {
 209         optional => 1,
 210         type => 'string',
 211         description => "Lock/unlock the VM.",
 212         enum => [qw(migrate backup snapshot rollback)],
 213     },
 214     cpulimit => {
 215         optional => 1,
 216         type => 'number',
 217         description => "Limit of CPU usage.",
 218         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 219         minimum => 0,
 220         maximum => 128,
 221         default => 0,
 222     },
 223     cpuunits => {
 224         optional => 1,
 225         type => 'integer',
 226         description => "CPU weight for a VM.",
 227         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 228         minimum => 2,
 229         maximum => 262144,
 230         default => 1024,
 231     },
 232     memory => {
 233         optional => 1,
 234         type => 'integer',
 235         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 236         minimum => 16,
 237         default => 512,
 238     },
 239     balloon => {
 240         optional => 1,
 241         type => 'integer',
 242         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 243         minimum => 0,
 244     },
 245     shares => {
 246         optional => 1,
 247         type => 'integer',
 248         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 249         minimum => 0,
 250         maximum => 50000,
 251         default => 1000,
 252     },
 253     keyboard => {
 254         optional => 1,
 255         type => 'string',
 256         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.".
 257                        "It should not be necessary to set it.",
 258         enum => PVE::Tools::kvmkeymaplist(),
 259         default => undef,
 260     },
 261     name => {
 262         optional => 1,
 263         type => 'string', format => 'dns-name',
 264         description => "Set a name for the VM. Only used on the configuration web interface.",
 265     },
 266     scsihw => {
 267         optional => 1,
 268         type => 'string',
 269         description => "SCSI controller model",
 270         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 271         default => 'lsi',
 272     },
 273     description => {
 274         optional => 1,
 275         type => 'string',
 276         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 277     },
 278     ostype => {
 279         optional => 1,
 280         type => 'string',
 281         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 282         description => "Specify guest operating system.",
 283         verbose_description => <<EODESC,
 284 Specify guest operating system. This is used to enable special
 285 optimization/features for specific operating systems:
 286
 287 [horizontal]
 288 other;; unspecified OS
 289 wxp;; Microsoft Windows XP
 290 w2k;; Microsoft Windows 2000
 291 w2k3;; Microsoft Windows 2003
 292 w2k8;; Microsoft Windows 2008
 293 wvista;; Microsoft Windows Vista
 294 win7;; Microsoft Windows 7
 295 win8;; Microsoft Windows 8/2012/2012r2
 296 win10;; Microsoft Windows 10/2016
 297 l24;; Linux 2.4 Kernel
 298 l26;; Linux 2.6/3.X Kernel
 299 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 300 EODESC
 301     },
 302     boot => {
 303         optional => 1,
 304         type => 'string',
 305         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 306         pattern => '[acdn]{1,4}',
 307         default => 'cdn',
 308     },
 309     bootdisk => {
 310         optional => 1,
 311         type => 'string', format => 'pve-qm-bootdisk',
 312         description => "Enable booting from specified disk.",
 313         pattern => '(ide|sata|scsi|virtio)\d+',
 314     },
 315     smp => {
 316         optional => 1,
 317         type => 'integer',
 318         description => "The number of CPUs. Please use option -sockets instead.",
 319         minimum => 1,
 320         default => 1,
 321     },
 322     sockets => {
 323         optional => 1,
 324         type => 'integer',
 325         description => "The number of CPU sockets.",
 326         minimum => 1,
 327         default => 1,
 328     },
 329     cores => {
 330         optional => 1,
 331         type => 'integer',
 332         description => "The number of cores per socket.",
 333         minimum => 1,
 334         default => 1,
 335     },
 336     numa => {
 337         optional => 1,
 338         type => 'boolean',
 339         description => "Enable/disable NUMA.",
 340         default => 0,
 341     },
 342     hugepages => {
 343         optional => 1,
 344         type => 'string',
 345         description => "Enable/disable hugepages memory.",
 346         enum => [qw(any 2 1024)],
 347     },
 348     vcpus => {
 349         optional => 1,
 350         type => 'integer',
 351         description => "Number of hotplugged vcpus.",
 352         minimum => 1,
 353         default => 0,
 354     },
 355     acpi => {
 356         optional => 1,
 357         type => 'boolean',
 358         description => "Enable/disable ACPI.",
 359         default => 1,
 360     },
 361     agent => {
 362         optional => 1,
 363         type => 'boolean',
 364         description => "Enable/disable Qemu GuestAgent.",
 365         default => 0,
 366     },
 367     kvm => {
 368         optional => 1,
 369         type => 'boolean',
 370         description => "Enable/disable KVM hardware virtualization.",
 371         default => 1,
 372     },
 373     tdf => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Enable/disable time drift fix.",
 377         default => 0,
 378     },
 379     localtime => {
 380         optional => 1,
 381         type => 'boolean',
 382         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 383     },
 384     freeze => {
 385         optional => 1,
 386         type => 'boolean',
 387         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 388     },
 389     vga => {
 390         optional => 1,
 391         type => 'string',
 392         description => "Select the VGA type.",
 393         verbose_description => "Select the VGA type. If you want to use high resolution" .
 394             " modes (>= 1280x1024x16) then you should use the options " .
 395             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 396             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 397             "display sever. For win* OS you can select how many independent " .
 398             "displays you want, Linux guests can add displays them self. " .
 399             "You can also run without any graphic card, using a serial device" .
 400             " as terminal.",
 401         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 402     },
 403     watchdog => {
 404         optional => 1,
 405         type => 'string', format => 'pve-qm-watchdog',
 406         description => "Create a virtual hardware watchdog device.",
 407         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 408             " (by a guest action), the watchdog must be periodically polled " .
 409             "by an agent inside the guest or else the watchdog will reset " .
 410             "the guest (or execute the respective action specified)",
 411     },
 412     startdate => {
 413         optional => 1,
 414         type => 'string',
 415         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 416         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 417         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 418         default => 'now',
 419     },
 420     startup =>  get_standard_option('pve-startup-order'),
 421     template => {
 422         optional => 1,
 423         type => 'boolean',
 424         description => "Enable/disable Template.",
 425         default => 0,
 426     },
 427     args => {
 428         optional => 1,
 429         type => 'string',
 430         description => "Arbitrary arguments passed to kvm.",
 431         verbose_description => <<EODESCR,
 432 Arbitrary arguments passed to kvm, for example:
 433
 434 args: -no-reboot -no-hpet
 435
 436 NOTE: this option is for experts only.
 437 EODESCR
 438     },
 439     tablet => {
 440         optional => 1,
 441         type => 'boolean',
 442         default => 1,
 443         description => "Enable/disable the USB tablet device.",
 444         verbose_description => "Enable/disable the USB tablet device. This device is " .
 445             "usually needed to allow absolute mouse positioning with VNC. " .
 446             "Else the mouse runs out of sync with normal VNC clients. " .
 447             "If you're running lots of console-only guests on one host, " .
 448             "you may consider disabling this to save some context switches. " .
 449             "This is turned off by default if you use spice (-vga=qxl).",
 450     },
 451     migrate_speed => {
 452         optional => 1,
 453         type => 'integer',
 454         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 455         minimum => 0,
 456         default => 0,
 457     },
 458     migrate_downtime => {
 459         optional => 1,
 460         type => 'number',
 461         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 462         minimum => 0,
 463         default => 0.1,
 464     },
 465     cdrom => {
 466         optional => 1,
 467         type => 'string', format => 'pve-qm-ide',
 468         typetext => '<volume>',
 469         description => "This is an alias for option -ide2",
 470     },
 471     cpu => {
 472         optional => 1,
 473         description => "Emulated CPU type.",
 474         type => 'string',
 475         format => $cpu_fmt,
 476     },
 477     parent => get_standard_option('pve-snapshot-name', {
 478         optional => 1,
 479         description => "Parent snapshot name. This is used internally, and should not be modified.",
 480     }),
 481     snaptime => {
 482         optional => 1,
 483         description => "Timestamp for snapshots.",
 484         type => 'integer',
 485         minimum => 0,
 486     },
 487     vmstate => {
 488         optional => 1,
 489         type => 'string', format => 'pve-volume-id',
 490         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 491     },
 492     vmstatestorage => get_standard_option('pve-storage-id', {
 493         description => "Default storage for VM state volumes/files.",
 494         optional => 1,
 495     }),
 496     machine => {
 497         description => "Specific the Qemu machine type.",
 498         type => 'string',
 499         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 500         maxLength => 40,
 501         optional => 1,
 502     },
 503     smbios1 => {
 504         description => "Specify SMBIOS type 1 fields.",
 505         type => 'string', format => 'pve-qm-smbios1',
 506         maxLength => 256,
 507         optional => 1,
 508     },
 509     protection => {
 510         optional => 1,
 511         type => 'boolean',
 512         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 513         default => 0,
 514     },
 515     bios => {
 516         optional => 1,
 517         type => 'string',
 518         enum => [ qw(seabios ovmf) ],
 519         description => "Select BIOS implementation.",
 520         default => 'seabios',
 521     },
 522 };
 523
 524 # what about other qemu settings ?
 525 #cpu => 'string',
 526 #machine => 'string',
 527 #fda => 'file',
 528 #fdb => 'file',
 529 #mtdblock => 'file',
 530 #sd => 'file',
 531 #pflash => 'file',
 532 #snapshot => 'bool',
 533 #bootp => 'file',
 534 ##tftp => 'dir',
 535 ##smb => 'dir',
 536 #kernel => 'file',
 537 #append => 'string',
 538 #initrd => 'file',
 539 ##soundhw => 'string',
 540
 541 while (my ($k, $v) = each %$confdesc) {
 542     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 543 }
 544
 545 my $MAX_IDE_DISKS = 4;
 546 my $MAX_SCSI_DISKS = 14;
 547 my $MAX_VIRTIO_DISKS = 16;
 548 my $MAX_SATA_DISKS = 6;
 549 my $MAX_USB_DEVICES = 5;
 550 my $MAX_NETS = 32;
 551 my $MAX_UNUSED_DISKS = 8;
 552 my $MAX_HOSTPCI_DEVICES = 4;
 553 my $MAX_SERIAL_PORTS = 4;
 554 my $MAX_PARALLEL_PORTS = 3;
 555 my $MAX_NUMA = 8;
 556
 557 my $numa_fmt = {
 558     cpus => {
 559         type => "string",
 560         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 561         description => "CPUs accessing this NUMA node.",
 562         format_description => "id[-id];...",
 563     },
 564     memory => {
 565         type => "number",
 566         description => "Amount of memory this NUMA node provides.",
 567         optional => 1,
 568     },
 569     hostnodes => {
 570         type => "string",
 571         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 572         description => "Host NUMA nodes to use.",
 573         format_description => "id[-id];...",
 574         optional => 1,
 575     },
 576     policy => {
 577         type => 'string',
 578         enum => [qw(preferred bind interleave)],
 579         description => "NUMA allocation policy.",
 580         optional => 1,
 581     },
 582 };
 583 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 584 my $numadesc = {
 585     optional => 1,
 586     type => 'string', format => $numa_fmt,
 587     description => "NUMA topology.",
 588 };
 589 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 590
 591 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 592     $confdesc->{"numa$i"} = $numadesc;
 593 }
 594
 595 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 596                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 597                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 598 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 599
 600 my $net_fmt_bridge_descr = <<__EOD__;
 601 Bridge to attach the network device to. The Proxmox VE standard bridge
 602 is called 'vmbr0'.
 603
 604 If you do not specify a bridge, we create a kvm user (NATed) network
 605 device, which provides DHCP and DNS services. The following addresses
 606 are used:
 607
 608  10.0.2.2   Gateway
 609  10.0.2.3   DNS Server
 610  10.0.2.4   SMB Server
 611
 612 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 613 __EOD__
 614
 615 my $net_fmt = {
 616     macaddr => {
 617         type => 'string',
 618         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 619         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 620         format_description => "XX:XX:XX:XX:XX:XX",
 621         optional => 1,
 622     },
 623     model => {
 624         type => 'string',
 625         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 626         enum => $nic_model_list,
 627         default_key => 1,
 628     },
 629     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 630     bridge => {
 631         type => 'string',
 632         description => $net_fmt_bridge_descr,
 633         format_description => 'bridge',
 634         optional => 1,
 635     },
 636     queues => {
 637         type => 'integer',
 638         minimum => 0, maximum => 16,
 639         description => 'Number of packet queues to be used on the device.',
 640         optional => 1,
 641     },
 642     rate => {
 643         type => 'number',
 644         minimum => 0,
 645         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 646         optional => 1,
 647     },
 648     tag => {
 649         type => 'integer',
 650         minimum => 1, maximum => 4094,
 651         description => 'VLAN tag to apply to packets on this interface.',
 652         optional => 1,
 653     },
 654     trunks => {
 655         type => 'string',
 656         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 657         description => 'VLAN trunks to pass through this interface.',
 658         format_description => 'vlanid[;vlanid...]',
 659         optional => 1,
 660     },
 661     firewall => {
 662         type => 'boolean',
 663         description => 'Whether this interface should be protected by the firewall.',
 664         optional => 1,
 665     },
 666     link_down => {
 667         type => 'boolean',
 668         description => 'Whether this interface should be disconnected (like pulling the plug).',
 669         optional => 1,
 670     },
 671 };
 672
 673 my $netdesc = {
 674     optional => 1,
 675     type => 'string', format => $net_fmt,
 676     description => "Specify network devices.",
 677 };
 678
 679 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 680
 681 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 682     $confdesc->{"net$i"} = $netdesc;
 683 }
 684
 685 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 686 sub verify_volume_id_or_qm_path {
 687     my ($volid, $noerr) = @_;
 688
 689     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 690         return $volid;
 691     }
 692
 693     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 694     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 695     if ($@) {
 696         return undef if $noerr;
 697         die $@;
 698     }
 699     return $volid;
 700 }
 701
 702 my $drivename_hash;
 703
 704 my %drivedesc_base = (
 705     volume => { alias => 'file' },
 706     file => {
 707         type => 'string',
 708         format => 'pve-volume-id-or-qm-path',
 709         default_key => 1,
 710         format_description => 'volume',
 711         description => "The drive's backing volume.",
 712     },
 713     media => {
 714         type => 'string',
 715         enum => [qw(cdrom disk)],
 716         description => "The drive's media type.",
 717         default => 'disk',
 718         optional => 1
 719     },
 720     cyls => {
 721         type => 'integer',
 722         description => "Force the drive's physical geometry to have a specific cylinder count.",
 723         optional => 1
 724     },
 725     heads => {
 726         type => 'integer',
 727         description => "Force the drive's physical geometry to have a specific head count.",
 728         optional => 1
 729     },
 730     secs => {
 731         type => 'integer',
 732         description => "Force the drive's physical geometry to have a specific sector count.",
 733         optional => 1
 734     },
 735     trans => {
 736         type => 'string',
 737         enum => [qw(none lba auto)],
 738         description => "Force disk geometry bios translation mode.",
 739         optional => 1,
 740     },
 741     snapshot => {
 742         type => 'boolean',
 743         description => "Whether the drive should be included when making snapshots.",
 744         optional => 1,
 745     },
 746     cache => {
 747         type => 'string',
 748         enum => [qw(none writethrough writeback unsafe directsync)],
 749         description => "The drive's cache mode",
 750         optional => 1,
 751     },
 752     format => get_standard_option('pve-qm-image-format'),
 753     size => {
 754         type => 'string',
 755         format => 'disk-size',
 756         format_description => 'DiskSize',
 757         description => "Disk size. This is purely informational and has no effect.",
 758         optional => 1,
 759     },
 760     backup => {
 761         type => 'boolean',
 762         description => "Whether the drive should be included when making backups.",
 763         optional => 1,
 764     },
 765     replicate => {
 766         type => 'boolean',
 767         description => 'Whether the drive should considered for replication jobs.',
 768         optional => 1,
 769         default => 1,
 770     },
 771     rerror => {
 772         type => 'string',
 773         enum => [qw(ignore report stop)],
 774         description => 'Read error action.',
 775         optional => 1,
 776     },
 777     werror => {
 778         type => 'string',
 779         enum => [qw(enospc ignore report stop)],
 780         description => 'Write error action.',
 781         optional => 1,
 782     },
 783     aio => {
 784         type => 'string',
 785         enum => [qw(native threads)],
 786         description => 'AIO type to use.',
 787         optional => 1,
 788     },
 789     discard => {
 790         type => 'string',
 791         enum => [qw(ignore on)],
 792         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 793         optional => 1,
 794     },
 795     detect_zeroes => {
 796         type => 'boolean',
 797         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 798         optional => 1,
 799     },
 800     serial => {
 801         type => 'string',
 802         format => 'urlencoded',
 803         format_description => 'serial',
 804         maxLength => 20*3, # *3 since it's %xx url enoded
 805         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 806         optional => 1,
 807     }
 808 );
 809
 810 my %iothread_fmt = ( iothread => {
 811         type => 'boolean',
 812         description => "Whether to use iothreads for this drive",
 813         optional => 1,
 814 });
 815
 816 my %model_fmt = (
 817     model => {
 818         type => 'string',
 819         format => 'urlencoded',
 820         format_description => 'model',
 821         maxLength => 40*3, # *3 since it's %xx url enoded
 822         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 823         optional => 1,
 824     },
 825 );
 826
 827 my %queues_fmt = (
 828     queues => {
 829         type => 'integer',
 830         description => "Number of queues.",
 831         minimum => 2,
 832         optional => 1
 833     }
 834 );
 835
 836 my %scsiblock_fmt = (
 837     scsiblock => {
 838         type => 'boolean',
 839         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 840         optional => 1,
 841         default => 0,
 842     },
 843 );
 844
 845 my $add_throttle_desc = sub {
 846     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 847     my $d = {
 848         type => $type,
 849         format_description => $unit,
 850         description => "Maximum $what in $longunit.",
 851         optional => 1,
 852     };
 853     $d->{minimum} = $minimum if defined($minimum);
 854     $drivedesc_base{$key} = $d;
 855 };
 856 # throughput: (leaky bucket)
 857 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 858 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 859 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 860 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 861 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 862 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 863 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 864 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 865 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 866
 867 # pools: (pool of IO before throttling starts taking effect)
 868 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 869 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 870 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 871 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 872 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 873 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 874
 875 # burst lengths
 876 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 877 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 878 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 879 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 880 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 881 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 882
 883 # legacy support
 884 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
 885 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
 886 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
 887 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
 888
 889 my $ide_fmt = {
 890     %drivedesc_base,
 891     %model_fmt,
 892 };
 893 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 894
 895 my $idedesc = {
 896     optional => 1,
 897     type => 'string', format => $ide_fmt,
 898     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 899 };
 900 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 901
 902 my $scsi_fmt = {
 903     %drivedesc_base,
 904     %iothread_fmt,
 905     %queues_fmt,
 906     %scsiblock_fmt,
 907 };
 908 my $scsidesc = {
 909     optional => 1,
 910     type => 'string', format => $scsi_fmt,
 911     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 912 };
 913 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 914
 915 my $sata_fmt = {
 916     %drivedesc_base,
 917 };
 918 my $satadesc = {
 919     optional => 1,
 920     type => 'string', format => $sata_fmt,
 921     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 922 };
 923 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 924
 925 my $virtio_fmt = {
 926     %drivedesc_base,
 927     %iothread_fmt,
 928 };
 929 my $virtiodesc = {
 930     optional => 1,
 931     type => 'string', format => $virtio_fmt,
 932     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 933 };
 934 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 935
 936 my $alldrive_fmt = {
 937     %drivedesc_base,
 938     %iothread_fmt,
 939     %model_fmt,
 940     %queues_fmt,
 941     %scsiblock_fmt,
 942 };
 943
 944 my $efidisk_fmt = {
 945     volume => { alias => 'file' },
 946     file => {
 947         type => 'string',
 948         format => 'pve-volume-id-or-qm-path',
 949         default_key => 1,
 950         format_description => 'volume',
 951         description => "The drive's backing volume.",
 952     },
 953     format => get_standard_option('pve-qm-image-format'),
 954     size => {
 955         type => 'string',
 956         format => 'disk-size',
 957         format_description => 'DiskSize',
 958         description => "Disk size. This is purely informational and has no effect.",
 959         optional => 1,
 960     },
 961 };
 962
 963 my $efidisk_desc = {
 964     optional => 1,
 965     type => 'string', format => $efidisk_fmt,
 966     description => "Configure a Disk for storing EFI vars",
 967 };
 968
 969 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 970
 971 my $usb_fmt = {
 972     host => {
 973         default_key => 1,
 974         type => 'string', format => 'pve-qm-usb-device',
 975         format_description => 'HOSTUSBDEVICE|spice',
 976         description => <<EODESCR,
 977 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 978
 979  'bus-port(.port)*' (decimal numbers) or
 980  'vendor_id:product_id' (hexadeciaml numbers) or
 981  'spice'
 982
 983 You can use the 'lsusb -t' command to list existing usb devices.
 984
 985 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 986
 987 The value 'spice' can be used to add a usb redirection devices for spice.
 988 EODESCR
 989     },
 990     usb3 => {
 991         optional => 1,
 992         type => 'boolean',
 993         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
 994         default => 0,
 995     },
 996 };
 997
 998 my $usbdesc = {
 999     optional => 1,
1000     type => 'string', format => $usb_fmt,
1001     description => "Configure an USB device (n is 0 to 4).",
1002 };
1003 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1004
1005 # NOTE: the match-groups of this regex are used in parse_hostpci
1006 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1007 my $hostpci_fmt = {
1008     host => {
1009         default_key => 1,
1010         type => 'string',
1011         pattern => qr/$PCIRE(;$PCIRE)*/,
1012         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1013         description => <<EODESCR,
1014 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1015 of PCI virtual functions of the host. HOSTPCIID syntax is:
1016
1017 'bus:dev.func' (hexadecimal numbers)
1018
1019 You can us the 'lspci' command to list existing PCI devices.
1020 EODESCR
1021     },
1022     rombar => {
1023         type => 'boolean',
1024         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1025         optional => 1,
1026         default => 1,
1027     },
1028     romfile => {
1029         type => 'string',
1030         pattern => '[^,;]+',
1031         format_description => 'string',
1032         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1033         optional => 1,
1034     },
1035     pcie => {
1036         type => 'boolean',
1037         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1038         optional => 1,
1039         default => 0,
1040     },
1041     'x-vga' => {
1042         type => 'boolean',
1043         description =>  "Enable vfio-vga device support.",
1044         optional => 1,
1045         default => 0,
1046     },
1047 };
1048 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1049
1050 my $hostpcidesc = {
1051         optional => 1,
1052         type => 'string', format => 'pve-qm-hostpci',
1053         description => "Map host PCI devices into guest.",
1054         verbose_description =>  <<EODESCR,
1055 Map host PCI devices into guest.
1056
1057 NOTE: This option allows direct access to host hardware. So it is no longer
1058 possible to migrate such machines - use with special care.
1059
1060 CAUTION: Experimental! User reported problems with this option.
1061 EODESCR
1062 };
1063 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1064
1065 my $serialdesc = {
1066         optional => 1,
1067         type => 'string',
1068         pattern => '(/dev/.+|socket)',
1069         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1070         verbose_description =>  <<EODESCR,
1071 Create a serial device inside the VM (n is 0 to 3), and pass through a
1072 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1073 host side (use 'qm terminal' to open a terminal connection).
1074
1075 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1076
1077 CAUTION: Experimental! User reported problems with this option.
1078 EODESCR
1079 };
1080
1081 my $paralleldesc= {
1082         optional => 1,
1083         type => 'string',
1084         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1085         description =>  "Map host parallel devices (n is 0 to 2).",
1086         verbose_description =>  <<EODESCR,
1087 Map host parallel devices (n is 0 to 2).
1088
1089 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1090
1091 CAUTION: Experimental! User reported problems with this option.
1092 EODESCR
1093 };
1094
1095 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1096     $confdesc->{"parallel$i"} = $paralleldesc;
1097 }
1098
1099 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1100     $confdesc->{"serial$i"} = $serialdesc;
1101 }
1102
1103 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1104     $confdesc->{"hostpci$i"} = $hostpcidesc;
1105 }
1106
1107 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1108     $drivename_hash->{"ide$i"} = 1;
1109     $confdesc->{"ide$i"} = $idedesc;
1110 }
1111
1112 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1113     $drivename_hash->{"sata$i"} = 1;
1114     $confdesc->{"sata$i"} = $satadesc;
1115 }
1116
1117 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1118     $drivename_hash->{"scsi$i"} = 1;
1119     $confdesc->{"scsi$i"} = $scsidesc ;
1120 }
1121
1122 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1123     $drivename_hash->{"virtio$i"} = 1;
1124     $confdesc->{"virtio$i"} = $virtiodesc;
1125 }
1126
1127 $drivename_hash->{efidisk0} = 1;
1128 $confdesc->{efidisk0} = $efidisk_desc;
1129
1130 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1131     $confdesc->{"usb$i"} = $usbdesc;
1132 }
1133
1134 my $unuseddesc = {
1135     optional => 1,
1136     type => 'string', format => 'pve-volume-id',
1137     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1138 };
1139
1140 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1141     $confdesc->{"unused$i"} = $unuseddesc;
1142 }
1143
1144 my $kvm_api_version = 0;
1145
1146 sub kvm_version {
1147
1148     return $kvm_api_version if $kvm_api_version;
1149
1150     my $fh = IO::File->new("</dev/kvm") ||
1151         return 0;
1152
1153     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1154         $kvm_api_version = $v;
1155     }
1156
1157     $fh->close();
1158
1159     return  $kvm_api_version;
1160 }
1161
1162 my $kvm_user_version;
1163
1164 sub kvm_user_version {
1165
1166     return $kvm_user_version if $kvm_user_version;
1167
1168     $kvm_user_version = 'unknown';
1169
1170     my $code = sub {
1171         my $line = shift;
1172         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1173             $kvm_user_version = $2;
1174         }
1175     };
1176
1177     eval { run_command("kvm -version", outfunc => $code); };
1178     warn $@ if $@;
1179
1180     return $kvm_user_version;
1181
1182 }
1183
1184 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1185
1186 sub valid_drive_names {
1187     # order is important - used to autoselect boot disk
1188     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1189             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1190             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1191             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1192             'efidisk0');
1193 }
1194
1195 sub is_valid_drivename {
1196     my $dev = shift;
1197
1198     return defined($drivename_hash->{$dev});
1199 }
1200
1201 sub option_exists {
1202     my $key = shift;
1203     return defined($confdesc->{$key});
1204 }
1205
1206 sub nic_models {
1207     return $nic_model_list;
1208 }
1209
1210 sub os_list_description {
1211
1212     return {
1213         other => 'Other',
1214         wxp => 'Windows XP',
1215         w2k => 'Windows 2000',
1216         w2k3 =>, 'Windows 2003',
1217         w2k8 => 'Windows 2008',
1218         wvista => 'Windows Vista',
1219         win7 => 'Windows 7',
1220         win8 => 'Windows 8/2012',
1221         win10 => 'Windows 10/2016',
1222         l24 => 'Linux 2.4',
1223         l26 => 'Linux 2.6',
1224     };
1225 }
1226
1227 my $cdrom_path;
1228
1229 sub get_cdrom_path {
1230
1231     return  $cdrom_path if $cdrom_path;
1232
1233     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1234     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1235     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1236 }
1237
1238 sub get_iso_path {
1239     my ($storecfg, $vmid, $cdrom) = @_;
1240
1241     if ($cdrom eq 'cdrom') {
1242         return get_cdrom_path();
1243     } elsif ($cdrom eq 'none') {
1244         return '';
1245     } elsif ($cdrom =~ m|^/|) {
1246         return $cdrom;
1247     } else {
1248         return PVE::Storage::path($storecfg, $cdrom);
1249     }
1250 }
1251
1252 # try to convert old style file names to volume IDs
1253 sub filename_to_volume_id {
1254     my ($vmid, $file, $media) = @_;
1255
1256     if (!($file eq 'none' || $file eq 'cdrom' ||
1257           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1258
1259         return undef if $file =~ m|/|;
1260
1261         if ($media && $media eq 'cdrom') {
1262             $file = "local:iso/$file";
1263         } else {
1264             $file = "local:$vmid/$file";
1265         }
1266     }
1267
1268     return $file;
1269 }
1270
1271 sub verify_media_type {
1272     my ($opt, $vtype, $media) = @_;
1273
1274     return if !$media;
1275
1276     my $etype;
1277     if ($media eq 'disk') {
1278         $etype = 'images';
1279     } elsif ($media eq 'cdrom') {
1280         $etype = 'iso';
1281     } else {
1282         die "internal error";
1283     }
1284
1285     return if ($vtype eq $etype);
1286
1287     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1288 }
1289
1290 sub cleanup_drive_path {
1291     my ($opt, $storecfg, $drive) = @_;
1292
1293     # try to convert filesystem paths to volume IDs
1294
1295     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1296         ($drive->{file} !~ m|^/dev/.+|) &&
1297         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1298         ($drive->{file} !~ m/^\d+$/)) {
1299         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1300         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1301         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1302         verify_media_type($opt, $vtype, $drive->{media});
1303         $drive->{file} = $volid;
1304     }
1305
1306     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1307 }
1308
1309 sub parse_hotplug_features {
1310     my ($data) = @_;
1311
1312     my $res = {};
1313
1314     return $res if $data eq '0';
1315
1316     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1317
1318     foreach my $feature (PVE::Tools::split_list($data)) {
1319         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1320             $res->{$1} = 1;
1321         } else {
1322             die "invalid hotplug feature '$feature'\n";
1323         }
1324     }
1325     return $res;
1326 }
1327
1328 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1329 sub pve_verify_hotplug_features {
1330     my ($value, $noerr) = @_;
1331
1332     return $value if parse_hotplug_features($value);
1333
1334     return undef if $noerr;
1335
1336     die "unable to parse hotplug option\n";
1337 }
1338
1339 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1340 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1341 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1342 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1343 #        [,iothread=on][,serial=serial][,model=model]
1344
1345 sub parse_drive {
1346     my ($key, $data) = @_;
1347
1348     my ($interface, $index);
1349
1350     if ($key =~ m/^([^\d]+)(\d+)$/) {
1351         $interface = $1;
1352         $index = $2;
1353     } else {
1354         return undef;
1355     }
1356
1357     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1358                                      : $confdesc->{$key}->{format};
1359     if (!$desc) {
1360         warn "invalid drive key: $key\n";
1361         return undef;
1362     }
1363     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1364     return undef if !$res;
1365     $res->{interface} = $interface;
1366     $res->{index} = $index;
1367
1368     my $error = 0;
1369     foreach my $opt (qw(bps bps_rd bps_wr)) {
1370         if (my $bps = defined(delete $res->{$opt})) {
1371             if (defined($res->{"m$opt"})) {
1372                 warn "both $opt and m$opt specified\n";
1373                 ++$error;
1374                 next;
1375             }
1376             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1377         }
1378     }
1379
1380     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1381     for my $requirement (
1382         [mbps_max => 'mbps'],
1383         [mbps_rd_max => 'mbps_rd'],
1384         [mbps_wr_max => 'mbps_wr'],
1385         [miops_max => 'miops'],
1386         [miops_rd_max => 'miops_rd'],
1387         [miops_wr_max => 'miops_wr'],
1388         [bps_max_length => 'mbps_max'],
1389         [bps_rd_max_length => 'mbps_rd_max'],
1390         [bps_wr_max_length => 'mbps_wr_max'],
1391         [iops_max_length => 'iops_max'],
1392         [iops_rd_max_length => 'iops_rd_max'],
1393         [iops_wr_max_length => 'iops_wr_max']) {
1394         my ($option, $requires) = @$requirement;
1395         if ($res->{$option} && !$res->{$requires}) {
1396             warn "$option requires $requires\n";
1397             ++$error;
1398         }
1399     }
1400
1401     return undef if $error;
1402
1403     return undef if $res->{mbps_rd} && $res->{mbps};
1404     return undef if $res->{mbps_wr} && $res->{mbps};
1405     return undef if $res->{iops_rd} && $res->{iops};
1406     return undef if $res->{iops_wr} && $res->{iops};
1407
1408     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1409         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1410         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1411         return undef if $res->{interface} eq 'virtio';
1412     }
1413
1414     if (my $size = $res->{size}) {
1415         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1416     }
1417
1418     return $res;
1419 }
1420
1421 sub print_drive {
1422     my ($vmid, $drive) = @_;
1423     my $data = { %$drive };
1424     delete $data->{$_} for qw(index interface);
1425     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1426 }
1427
1428 sub scsi_inquiry {
1429     my($fh, $noerr) = @_;
1430
1431     my $SG_IO = 0x2285;
1432     my $SG_GET_VERSION_NUM = 0x2282;
1433
1434     my $versionbuf = "\x00" x 8;
1435     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1436     if (!$ret) {
1437         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1438         return undef;
1439     }
1440     my $version = unpack("I", $versionbuf);
1441     if ($version < 30000) {
1442         die "scsi generic interface too old\n"  if !$noerr;
1443         return undef;
1444     }
1445
1446     my $buf = "\x00" x 36;
1447     my $sensebuf = "\x00" x 8;
1448     my $cmd = pack("C x3 C x1", 0x12, 36);
1449
1450     # see /usr/include/scsi/sg.h
1451     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1452
1453     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1454                       length($sensebuf), 0, length($buf), $buf,
1455                       $cmd, $sensebuf, 6000);
1456
1457     $ret = ioctl($fh, $SG_IO, $packet);
1458     if (!$ret) {
1459         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1460         return undef;
1461     }
1462
1463     my @res = unpack($sg_io_hdr_t, $packet);
1464     if ($res[17] || $res[18]) {
1465         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1466         return undef;
1467     }
1468
1469     my $res = {};
1470     (my $byte0, my $byte1, $res->{vendor},
1471      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1472
1473     $res->{removable} = $byte1 & 128 ? 1 : 0;
1474     $res->{type} = $byte0 & 31;
1475
1476     return $res;
1477 }
1478
1479 sub path_is_scsi {
1480     my ($path) = @_;
1481
1482     my $fh = IO::File->new("+<$path") || return undef;
1483     my $res = scsi_inquiry($fh, 1);
1484     close($fh);
1485
1486     return $res;
1487 }
1488
1489 sub machine_type_is_q35 {
1490     my ($conf) = @_;
1491
1492     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1493 }
1494
1495 sub print_tabletdevice_full {
1496     my ($conf) = @_;
1497
1498     my $q35 = machine_type_is_q35($conf);
1499
1500     # we use uhci for old VMs because tablet driver was buggy in older qemu
1501     my $usbbus = $q35 ? "ehci" : "uhci";
1502
1503     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1504 }
1505
1506 sub print_drivedevice_full {
1507     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1508
1509     my $device = '';
1510     my $maxdev = 0;
1511
1512     if ($drive->{interface} eq 'virtio') {
1513         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1514         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1515         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1516     } elsif ($drive->{interface} eq 'scsi') {
1517
1518         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1519         my $unit = $drive->{index} % $maxdev;
1520         my $devicetype = 'hd';
1521         my $path = '';
1522         if (drive_is_cdrom($drive)) {
1523             $devicetype = 'cd';
1524         } else {
1525             if ($drive->{file} =~ m|^/|) {
1526                 $path = $drive->{file};
1527                 if (my $info = path_is_scsi($path)) {
1528                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1529                         $devicetype = 'block';
1530                     } elsif ($info->{type} == 1) { # tape
1531                         $devicetype = 'generic';
1532                     }
1533                 }
1534             } else {
1535                  $path = PVE::Storage::path($storecfg, $drive->{file});
1536             }
1537
1538             if($path =~ m/^iscsi\:\/\//){
1539                 $devicetype = 'generic';
1540             }
1541         }
1542
1543         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1544             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1545         } else {
1546             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1547         }
1548
1549     } elsif ($drive->{interface} eq 'ide'){
1550         $maxdev = 2;
1551         my $controller = int($drive->{index} / $maxdev);
1552         my $unit = $drive->{index} % $maxdev;
1553         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1554
1555         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1556         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1557             $model = URI::Escape::uri_unescape($model);
1558             $device .= ",model=$model";
1559         }
1560     } elsif ($drive->{interface} eq 'sata'){
1561         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1562         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1563         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1564     } elsif ($drive->{interface} eq 'usb') {
1565         die "implement me";
1566         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1567     } else {
1568         die "unsupported interface type";
1569     }
1570
1571     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1572
1573     return $device;
1574 }
1575
1576 sub get_initiator_name {
1577     my $initiator;
1578
1579     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1580     while (defined(my $line = <$fh>)) {
1581         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1582         $initiator = $1;
1583         last;
1584     }
1585     $fh->close();
1586
1587     return $initiator;
1588 }
1589
1590 sub print_drive_full {
1591     my ($storecfg, $vmid, $drive) = @_;
1592
1593     my $path;
1594     my $volid = $drive->{file};
1595     my $format;
1596
1597     if (drive_is_cdrom($drive)) {
1598         $path = get_iso_path($storecfg, $vmid, $volid);
1599     } else {
1600         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1601         if ($storeid) {
1602             $path = PVE::Storage::path($storecfg, $volid);
1603             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1604             $format = qemu_img_format($scfg, $volname);
1605         } else {
1606             $path = $volid;
1607             $format = "raw";
1608         }
1609    }
1610
1611     my $opts = '';
1612     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard);
1613     foreach my $o (@qemu_drive_options) {
1614         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1615     }
1616     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1617         my ($dir, $qmpname) = @$type;
1618         if (my $v = $drive->{"mbps$dir"}) {
1619             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1620         }
1621         if (my $v = $drive->{"mbps${dir}_max"}) {
1622             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1623         }
1624         if (my $v = $drive->{"bps${dir}_max_length"}) {
1625             $opts .= ",throttling.bps$qmpname-max-length=$v";
1626         }
1627         if (my $v = $drive->{"iops${dir}"}) {
1628             $opts .= ",throttling.iops$qmpname=$v";
1629         }
1630         if (my $v = $drive->{"iops${dir}_max"}) {
1631             $opts .= ",throttling.iops$qmpname-max=$v";
1632         }
1633         if (my $v = $drive->{"iops${dir}_max_length"}) {
1634             $opts .= ",throttling.iops$qmpname-max-length=$v";
1635         }
1636     }
1637
1638     if (my $serial = $drive->{serial}) {
1639         $serial = URI::Escape::uri_unescape($serial);
1640         $opts .= ",serial=$serial";
1641     }
1642
1643     $opts .= ",format=$format" if $format && !$drive->{format};
1644
1645     my $cache_direct = 0;
1646
1647     if (my $cache = $drive->{cache}) {
1648         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1649     } elsif (!drive_is_cdrom($drive)) {
1650         $opts .= ",cache=none";
1651         $cache_direct = 1;
1652     }
1653
1654     # aio native works only with O_DIRECT
1655     if (!$drive->{aio}) {
1656         if($cache_direct) {
1657             $opts .= ",aio=native";
1658         } else {
1659             $opts .= ",aio=threads";
1660         }
1661     }
1662
1663     if (!drive_is_cdrom($drive)) {
1664         my $detectzeroes;
1665         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1666             $detectzeroes = 'off';
1667         } elsif ($drive->{discard}) {
1668             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1669         } else {
1670             # This used to be our default with discard not being specified:
1671             $detectzeroes = 'on';
1672         }
1673         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1674     }
1675
1676     my $pathinfo = $path ? "file=$path," : '';
1677
1678     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1679 }
1680
1681 sub print_netdevice_full {
1682     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1683
1684     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1685
1686     my $device = $net->{model};
1687     if ($net->{model} eq 'virtio') {
1688          $device = 'virtio-net-pci';
1689      };
1690
1691     my $pciaddr = print_pci_addr("$netid", $bridges);
1692     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1693     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1694         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1695         my $vectors = $net->{queues} * 2 + 2;
1696         $tmpstr .= ",vectors=$vectors,mq=on";
1697     }
1698     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1699
1700     if ($use_old_bios_files) {
1701         my $romfile;
1702         if ($device eq 'virtio-net-pci') {
1703             $romfile = 'pxe-virtio.rom';
1704         } elsif ($device eq 'e1000') {
1705             $romfile = 'pxe-e1000.rom';
1706         } elsif ($device eq 'ne2k') {
1707             $romfile = 'pxe-ne2k_pci.rom';
1708         } elsif ($device eq 'pcnet') {
1709             $romfile = 'pxe-pcnet.rom';
1710         } elsif ($device eq 'rtl8139') {
1711             $romfile = 'pxe-rtl8139.rom';
1712         }
1713         $tmpstr .= ",romfile=$romfile" if $romfile;
1714     }
1715
1716     return $tmpstr;
1717 }
1718
1719 sub print_netdev_full {
1720     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1721
1722     my $i = '';
1723     if ($netid =~ m/^net(\d+)$/) {
1724         $i = int($1);
1725     }
1726
1727     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1728
1729     my $ifname = "tap${vmid}i$i";
1730
1731     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1732     die "interface name '$ifname' is too long (max 15 character)\n"
1733         if length($ifname) >= 16;
1734
1735     my $vhostparam = '';
1736     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1737
1738     my $vmname = $conf->{name} || "vm$vmid";
1739
1740     my $netdev = "";
1741     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1742
1743     if ($net->{bridge}) {
1744         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1745     } else {
1746         $netdev = "type=user,id=$netid,hostname=$vmname";
1747     }
1748
1749     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1750
1751     return $netdev;
1752 }
1753
1754
1755 sub print_cpu_device {
1756     my ($conf, $id) = @_;
1757
1758     my $kvm = $conf->{kvm} // 1;
1759     my $cpu = $kvm ? "kvm64" : "qemu64";
1760     if (my $cputype = $conf->{cpu}) {
1761         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1762             or die "Cannot parse cpu description: $cputype\n";
1763         $cpu = $cpuconf->{cputype};
1764     }
1765
1766     my $cores = $conf->{cores} || 1;
1767
1768     my $current_core = ($id - 1) % $cores;
1769     my $current_socket = int(($id - 1 - $current_core)/$cores);
1770
1771     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1772 }
1773
1774 sub drive_is_cdrom {
1775     my ($drive) = @_;
1776
1777     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1778
1779 }
1780
1781 sub parse_number_sets {
1782     my ($set) = @_;
1783     my $res = [];
1784     foreach my $part (split(/;/, $set)) {
1785         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1786             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1787             push @$res, [ $1, $2 ];
1788         } else {
1789             die "invalid range: $part\n";
1790         }
1791     }
1792     return $res;
1793 }
1794
1795 sub parse_numa {
1796     my ($data) = @_;
1797
1798     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1799     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1800     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1801     return $res;
1802 }
1803
1804 sub parse_hostpci {
1805     my ($value) = @_;
1806
1807     return undef if !$value;
1808
1809     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1810
1811     my @idlist = split(/;/, $res->{host});
1812     delete $res->{host};
1813     foreach my $id (@idlist) {
1814         if ($id =~ /^$PCIRE$/) {
1815             if (defined($2)) {
1816                 push @{$res->{pciid}}, { id => $1, function => $2 };
1817             } else {
1818                 my $pcidevices = lspci($1);
1819                 $res->{pciid} = $pcidevices->{$1};
1820             }
1821         } else {
1822             # should have been caught by parse_property_string already
1823             die "failed to parse PCI id: $id\n";
1824         }
1825     }
1826     return $res;
1827 }
1828
1829 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1830 sub parse_net {
1831     my ($data) = @_;
1832
1833     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1834     if ($@) {
1835         warn $@;
1836         return undef;
1837     }
1838     if (!defined($res->{macaddr})) {
1839         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1840         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1841     }
1842     return $res;
1843 }
1844
1845 sub print_net {
1846     my $net = shift;
1847
1848     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1849 }
1850
1851 sub add_random_macs {
1852     my ($settings) = @_;
1853
1854     foreach my $opt (keys %$settings) {
1855         next if $opt !~ m/^net(\d+)$/;
1856         my $net = parse_net($settings->{$opt});
1857         next if !$net;
1858         $settings->{$opt} = print_net($net);
1859     }
1860 }
1861
1862 sub vm_is_volid_owner {
1863     my ($storecfg, $vmid, $volid) = @_;
1864
1865     if ($volid !~  m|^/|) {
1866         my ($path, $owner);
1867         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1868         if ($owner && ($owner == $vmid)) {
1869             return 1;
1870         }
1871     }
1872
1873     return undef;
1874 }
1875
1876 sub split_flagged_list {
1877     my $text = shift || '';
1878     $text =~ s/[,;]/ /g;
1879     $text =~ s/^\s+//;
1880     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1881 }
1882
1883 sub join_flagged_list {
1884     my ($how, $lst) = @_;
1885     join $how, map { $lst->{$_} . $_ } keys %$lst;
1886 }
1887
1888 sub vmconfig_delete_pending_option {
1889     my ($conf, $key, $force) = @_;
1890
1891     delete $conf->{pending}->{$key};
1892     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1893     $pending_delete_hash->{$key} = $force ? '!' : '';
1894     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1895 }
1896
1897 sub vmconfig_undelete_pending_option {
1898     my ($conf, $key) = @_;
1899
1900     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1901     delete $pending_delete_hash->{$key};
1902
1903     if (%$pending_delete_hash) {
1904         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1905     } else {
1906         delete $conf->{pending}->{delete};
1907     }
1908 }
1909
1910 sub vmconfig_register_unused_drive {
1911     my ($storecfg, $vmid, $conf, $drive) = @_;
1912
1913     if (!drive_is_cdrom($drive)) {
1914         my $volid = $drive->{file};
1915         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1916             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1917         }
1918     }
1919 }
1920
1921 sub vmconfig_cleanup_pending {
1922     my ($conf) = @_;
1923
1924     # remove pending changes when nothing changed
1925     my $changes;
1926     foreach my $opt (keys %{$conf->{pending}}) {
1927         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1928             $changes = 1;
1929             delete $conf->{pending}->{$opt};
1930         }
1931     }
1932
1933     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1934     my $pending_delete_hash = {};
1935     while (my ($opt, $force) = each %$current_delete_hash) {
1936         if (defined($conf->{$opt})) {
1937             $pending_delete_hash->{$opt} = $force;
1938         } else {
1939             $changes = 1;
1940         }
1941     }
1942
1943     if (%$pending_delete_hash) {
1944         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1945     } else {
1946         delete $conf->{pending}->{delete};
1947     }
1948
1949     return $changes;
1950 }
1951
1952 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1953 my $smbios1_fmt = {
1954     uuid => {
1955         type => 'string',
1956         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1957         format_description => 'UUID',
1958         description => "Set SMBIOS1 UUID.",
1959         optional => 1,
1960     },
1961     version => {
1962         type => 'string',
1963         pattern => '\S+',
1964         format_description => 'string',
1965         description => "Set SMBIOS1 version.",
1966         optional => 1,
1967     },
1968     serial => {
1969         type => 'string',
1970         pattern => '\S+',
1971         format_description => 'string',
1972         description => "Set SMBIOS1 serial number.",
1973         optional => 1,
1974     },
1975     manufacturer => {
1976         type => 'string',
1977         pattern => '\S+',
1978         format_description => 'string',
1979         description => "Set SMBIOS1 manufacturer.",
1980         optional => 1,
1981     },
1982     product => {
1983         type => 'string',
1984         pattern => '\S+',
1985         format_description => 'string',
1986         description => "Set SMBIOS1 product ID.",
1987         optional => 1,
1988     },
1989     sku => {
1990         type => 'string',
1991         pattern => '\S+',
1992         format_description => 'string',
1993         description => "Set SMBIOS1 SKU string.",
1994         optional => 1,
1995     },
1996     family => {
1997         type => 'string',
1998         pattern => '\S+',
1999         format_description => 'string',
2000         description => "Set SMBIOS1 family string.",
2001         optional => 1,
2002     },
2003 };
2004
2005 sub parse_smbios1 {
2006     my ($data) = @_;
2007
2008     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2009     warn $@ if $@;
2010     return $res;
2011 }
2012
2013 sub print_smbios1 {
2014     my ($smbios1) = @_;
2015     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2016 }
2017
2018 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2019
2020 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2021 sub verify_bootdisk {
2022     my ($value, $noerr) = @_;
2023
2024     return $value if is_valid_drivename($value);
2025
2026     return undef if $noerr;
2027
2028     die "invalid boot disk '$value'\n";
2029 }
2030
2031 sub parse_watchdog {
2032     my ($value) = @_;
2033
2034     return undef if !$value;
2035
2036     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2037     warn $@ if $@;
2038     return $res;
2039 }
2040
2041 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2042 sub verify_usb_device {
2043     my ($value, $noerr) = @_;
2044
2045     return $value if parse_usb_device($value);
2046
2047     return undef if $noerr;
2048
2049     die "unable to parse usb device\n";
2050 }
2051
2052 # add JSON properties for create and set function
2053 sub json_config_properties {
2054     my $prop = shift;
2055
2056     foreach my $opt (keys %$confdesc) {
2057         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2058         $prop->{$opt} = $confdesc->{$opt};
2059     }
2060
2061     return $prop;
2062 }
2063
2064 sub check_type {
2065     my ($key, $value) = @_;
2066
2067     die "unknown setting '$key'\n" if !$confdesc->{$key};
2068
2069     my $type = $confdesc->{$key}->{type};
2070
2071     if (!defined($value)) {
2072         die "got undefined value\n";
2073     }
2074
2075     if ($value =~ m/[\n\r]/) {
2076         die "property contains a line feed\n";
2077     }
2078
2079     if ($type eq 'boolean') {
2080         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2081         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2082         die "type check ('boolean') failed - got '$value'\n";
2083     } elsif ($type eq 'integer') {
2084         return int($1) if $value =~ m/^(\d+)$/;
2085         die "type check ('integer') failed - got '$value'\n";
2086     } elsif ($type eq 'number') {
2087         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2088         die "type check ('number') failed - got '$value'\n";
2089     } elsif ($type eq 'string') {
2090         if (my $fmt = $confdesc->{$key}->{format}) {
2091             PVE::JSONSchema::check_format($fmt, $value);
2092             return $value;
2093         }
2094         $value =~ s/^\"(.*)\"$/$1/;
2095         return $value;
2096     } else {
2097         die "internal error"
2098     }
2099 }
2100
2101 sub check_iommu_support{
2102     #fixme : need to check IOMMU support
2103     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2104
2105     my $iommu=1;
2106     return $iommu;
2107
2108 }
2109
2110 sub touch_config {
2111     my ($vmid) = @_;
2112
2113     my $conf = PVE::QemuConfig->config_file($vmid);
2114     utime undef, undef, $conf;
2115 }
2116
2117 sub destroy_vm {
2118     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2119
2120     my $conffile = PVE::QemuConfig->config_file($vmid);
2121
2122     my $conf = PVE::QemuConfig->load_config($vmid);
2123
2124     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2125
2126     if ($conf->{template}) {
2127         # check if any base image is still used by a linked clone
2128         foreach_drive($conf, sub {
2129                 my ($ds, $drive) = @_;
2130
2131                 return if drive_is_cdrom($drive);
2132
2133                 my $volid = $drive->{file};
2134
2135                 return if !$volid || $volid =~ m|^/|;
2136
2137                 die "base volume '$volid' is still in use by linked cloned\n"
2138                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2139
2140         });
2141     }
2142
2143     # only remove disks owned by this VM
2144     foreach_drive($conf, sub {
2145         my ($ds, $drive) = @_;
2146
2147         return if drive_is_cdrom($drive);
2148
2149         my $volid = $drive->{file};
2150
2151         return if !$volid || $volid =~ m|^/|;
2152
2153         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2154         return if !$path || !$owner || ($owner != $vmid);
2155
2156         eval {
2157             PVE::Storage::vdisk_free($storecfg, $volid);
2158         };
2159         warn "Could not remove disk '$volid', check manually: $@" if $@;
2160
2161     });
2162
2163     if ($keep_empty_config) {
2164         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2165     } else {
2166         unlink $conffile;
2167     }
2168
2169     # also remove unused disk
2170     eval {
2171         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2172
2173         eval {
2174             PVE::Storage::foreach_volid($dl, sub {
2175                 my ($volid, $sid, $volname, $d) = @_;
2176                 PVE::Storage::vdisk_free($storecfg, $volid);
2177             });
2178         };
2179         warn $@ if $@;
2180
2181     };
2182     warn $@ if $@;
2183 }
2184
2185 sub parse_vm_config {
2186     my ($filename, $raw) = @_;
2187
2188     return undef if !defined($raw);
2189
2190     my $res = {
2191         digest => Digest::SHA::sha1_hex($raw),
2192         snapshots => {},
2193         pending => {},
2194     };
2195
2196     $filename =~ m|/qemu-server/(\d+)\.conf$|
2197         || die "got strange filename '$filename'";
2198
2199     my $vmid = $1;
2200
2201     my $conf = $res;
2202     my $descr;
2203     my $section = '';
2204
2205     my @lines = split(/\n/, $raw);
2206     foreach my $line (@lines) {
2207         next if $line =~ m/^\s*$/;
2208
2209         if ($line =~ m/^\[PENDING\]\s*$/i) {
2210             $section = 'pending';
2211             if (defined($descr)) {
2212                 $descr =~ s/\s+$//;
2213                 $conf->{description} = $descr;
2214             }
2215             $descr = undef;
2216             $conf = $res->{$section} = {};
2217             next;
2218
2219         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2220             $section = $1;
2221             if (defined($descr)) {
2222                 $descr =~ s/\s+$//;
2223                 $conf->{description} = $descr;
2224             }
2225             $descr = undef;
2226             $conf = $res->{snapshots}->{$section} = {};
2227             next;
2228         }
2229
2230         if ($line =~ m/^\#(.*)\s*$/) {
2231             $descr = '' if !defined($descr);
2232             $descr .= PVE::Tools::decode_text($1) . "\n";
2233             next;
2234         }
2235
2236         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2237             $descr = '' if !defined($descr);
2238             $descr .= PVE::Tools::decode_text($2);
2239         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2240             $conf->{snapstate} = $1;
2241         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2242             my $key = $1;
2243             my $value = $2;
2244             $conf->{$key} = $value;
2245         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2246             my $value = $1;
2247             if ($section eq 'pending') {
2248                 $conf->{delete} = $value; # we parse this later
2249             } else {
2250                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2251             }
2252         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2253             my $key = $1;
2254             my $value = $2;
2255             eval { $value = check_type($key, $value); };
2256             if ($@) {
2257                 warn "vm $vmid - unable to parse value of '$key' - $@";
2258             } else {
2259                 $key = 'ide2' if $key eq 'cdrom';
2260                 my $fmt = $confdesc->{$key}->{format};
2261                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2262                     my $v = parse_drive($key, $value);
2263                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2264                         $v->{file} = $volid;
2265                         $value = print_drive($vmid, $v);
2266                     } else {
2267                         warn "vm $vmid - unable to parse value of '$key'\n";
2268                         next;
2269                     }
2270                 }
2271
2272                 $conf->{$key} = $value;
2273             }
2274         }
2275     }
2276
2277     if (defined($descr)) {
2278         $descr =~ s/\s+$//;
2279         $conf->{description} = $descr;
2280     }
2281     delete $res->{snapstate}; # just to be sure
2282
2283     return $res;
2284 }
2285
2286 sub write_vm_config {
2287     my ($filename, $conf) = @_;
2288
2289     delete $conf->{snapstate}; # just to be sure
2290
2291     if ($conf->{cdrom}) {
2292         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2293         $conf->{ide2} = $conf->{cdrom};
2294         delete $conf->{cdrom};
2295     }
2296
2297     # we do not use 'smp' any longer
2298     if ($conf->{sockets}) {
2299         delete $conf->{smp};
2300     } elsif ($conf->{smp}) {
2301         $conf->{sockets} = $conf->{smp};
2302         delete $conf->{cores};
2303         delete $conf->{smp};
2304     }
2305
2306     my $used_volids = {};
2307
2308     my $cleanup_config = sub {
2309         my ($cref, $pending, $snapname) = @_;
2310
2311         foreach my $key (keys %$cref) {
2312             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2313                 $key eq 'snapstate' || $key eq 'pending';
2314             my $value = $cref->{$key};
2315             if ($key eq 'delete') {
2316                 die "propertry 'delete' is only allowed in [PENDING]\n"
2317                     if !$pending;
2318                 # fixme: check syntax?
2319                 next;
2320             }
2321             eval { $value = check_type($key, $value); };
2322             die "unable to parse value of '$key' - $@" if $@;
2323
2324             $cref->{$key} = $value;
2325
2326             if (!$snapname && is_valid_drivename($key)) {
2327                 my $drive = parse_drive($key, $value);
2328                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2329             }
2330         }
2331     };
2332
2333     &$cleanup_config($conf);
2334
2335     &$cleanup_config($conf->{pending}, 1);
2336
2337     foreach my $snapname (keys %{$conf->{snapshots}}) {
2338         die "internal error" if $snapname eq 'pending';
2339         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2340     }
2341
2342     # remove 'unusedX' settings if we re-add a volume
2343     foreach my $key (keys %$conf) {
2344         my $value = $conf->{$key};
2345         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2346             delete $conf->{$key};
2347         }
2348     }
2349
2350     my $generate_raw_config = sub {
2351         my ($conf, $pending) = @_;
2352
2353         my $raw = '';
2354
2355         # add description as comment to top of file
2356         if (defined(my $descr = $conf->{description})) {
2357             if ($descr) {
2358                 foreach my $cl (split(/\n/, $descr)) {
2359                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2360                 }
2361             } else {
2362                 $raw .= "#\n" if $pending;
2363             }
2364         }
2365
2366         foreach my $key (sort keys %$conf) {
2367             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2368             $raw .= "$key: $conf->{$key}\n";
2369         }
2370         return $raw;
2371     };
2372
2373     my $raw = &$generate_raw_config($conf);
2374
2375     if (scalar(keys %{$conf->{pending}})){
2376         $raw .= "\n[PENDING]\n";
2377         $raw .= &$generate_raw_config($conf->{pending}, 1);
2378     }
2379
2380     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2381         $raw .= "\n[$snapname]\n";
2382         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2383     }
2384
2385     return $raw;
2386 }
2387
2388 sub load_defaults {
2389
2390     my $res = {};
2391
2392     # we use static defaults from our JSON schema configuration
2393     foreach my $key (keys %$confdesc) {
2394         if (defined(my $default = $confdesc->{$key}->{default})) {
2395             $res->{$key} = $default;
2396         }
2397     }
2398
2399     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2400     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2401
2402     return $res;
2403 }
2404
2405 sub config_list {
2406     my $vmlist = PVE::Cluster::get_vmlist();
2407     my $res = {};
2408     return $res if !$vmlist || !$vmlist->{ids};
2409     my $ids = $vmlist->{ids};
2410
2411     foreach my $vmid (keys %$ids) {
2412         my $d = $ids->{$vmid};
2413         next if !$d->{node} || $d->{node} ne $nodename;
2414         next if !$d->{type} || $d->{type} ne 'qemu';
2415         $res->{$vmid}->{exists} = 1;
2416     }
2417     return $res;
2418 }
2419
2420 # test if VM uses local resources (to prevent migration)
2421 sub check_local_resources {
2422     my ($conf, $noerr) = @_;
2423
2424     my $loc_res = 0;
2425
2426     $loc_res = 1 if $conf->{hostusb}; # old syntax
2427     $loc_res = 1 if $conf->{hostpci}; # old syntax
2428
2429     foreach my $k (keys %$conf) {
2430         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2431         # sockets are safe: they will recreated be on the target side post-migrate
2432         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2433         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2434     }
2435
2436     die "VM uses local resources\n" if $loc_res && !$noerr;
2437
2438     return $loc_res;
2439 }
2440
2441 # check if used storages are available on all nodes (use by migrate)
2442 sub check_storage_availability {
2443     my ($storecfg, $conf, $node) = @_;
2444
2445     foreach_drive($conf, sub {
2446         my ($ds, $drive) = @_;
2447
2448         my $volid = $drive->{file};
2449         return if !$volid;
2450
2451         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2452         return if !$sid;
2453
2454         # check if storage is available on both nodes
2455         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2456         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2457    });
2458 }
2459
2460 # list nodes where all VM images are available (used by has_feature API)
2461 sub shared_nodes {
2462     my ($conf, $storecfg) = @_;
2463
2464     my $nodelist = PVE::Cluster::get_nodelist();
2465     my $nodehash = { map { $_ => 1 } @$nodelist };
2466     my $nodename = PVE::INotify::nodename();
2467
2468     foreach_drive($conf, sub {
2469         my ($ds, $drive) = @_;
2470
2471         my $volid = $drive->{file};
2472         return if !$volid;
2473
2474         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2475         if ($storeid) {
2476             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2477             if ($scfg->{disable}) {
2478                 $nodehash = {};
2479             } elsif (my $avail = $scfg->{nodes}) {
2480                 foreach my $node (keys %$nodehash) {
2481                     delete $nodehash->{$node} if !$avail->{$node};
2482                 }
2483             } elsif (!$scfg->{shared}) {
2484                 foreach my $node (keys %$nodehash) {
2485                     delete $nodehash->{$node} if $node ne $nodename
2486                 }
2487             }
2488         }
2489     });
2490
2491     return $nodehash
2492 }
2493
2494 sub check_cmdline {
2495     my ($pidfile, $pid) = @_;
2496
2497     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2498     if (defined($fh)) {
2499         my $line = <$fh>;
2500         $fh->close;
2501         return undef if !$line;
2502         my @param = split(/\0/, $line);
2503
2504         my $cmd = $param[0];
2505         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2506
2507         for (my $i = 0; $i < scalar (@param); $i++) {
2508             my $p = $param[$i];
2509             next if !$p;
2510             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2511                 my $p = $param[$i+1];
2512                 return 1 if $p && ($p eq $pidfile);
2513                 return undef;
2514             }
2515         }
2516     }
2517     return undef;
2518 }
2519
2520 sub check_running {
2521     my ($vmid, $nocheck, $node) = @_;
2522
2523     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2524
2525     die "unable to find configuration file for VM $vmid - no such machine\n"
2526         if !$nocheck && ! -f $filename;
2527
2528     my $pidfile = pidfile_name($vmid);
2529
2530     if (my $fd = IO::File->new("<$pidfile")) {
2531         my $st = stat($fd);
2532         my $line = <$fd>;
2533         close($fd);
2534
2535         my $mtime = $st->mtime;
2536         if ($mtime > time()) {
2537             warn "file '$filename' modified in future\n";
2538         }
2539
2540         if ($line =~ m/^(\d+)$/) {
2541             my $pid = $1;
2542             if (check_cmdline($pidfile, $pid)) {
2543                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2544                     return $pid;
2545                 }
2546             }
2547         }
2548     }
2549
2550     return undef;
2551 }
2552
2553 sub vzlist {
2554
2555     my $vzlist = config_list();
2556
2557     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2558
2559     while (defined(my $de = $fd->read)) {
2560         next if $de !~ m/^(\d+)\.pid$/;
2561         my $vmid = $1;
2562         next if !defined($vzlist->{$vmid});
2563         if (my $pid = check_running($vmid)) {
2564             $vzlist->{$vmid}->{pid} = $pid;
2565         }
2566     }
2567
2568     return $vzlist;
2569 }
2570
2571 sub disksize {
2572     my ($storecfg, $conf) = @_;
2573
2574     my $bootdisk = $conf->{bootdisk};
2575     return undef if !$bootdisk;
2576     return undef if !is_valid_drivename($bootdisk);
2577
2578     return undef if !$conf->{$bootdisk};
2579
2580     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2581     return undef if !defined($drive);
2582
2583     return undef if drive_is_cdrom($drive);
2584
2585     my $volid = $drive->{file};
2586     return undef if !$volid;
2587
2588     return $drive->{size};
2589 }
2590
2591 my $last_proc_pid_stat;
2592
2593 # get VM status information
2594 # This must be fast and should not block ($full == false)
2595 # We only query KVM using QMP if $full == true (this can be slow)
2596 sub vmstatus {
2597     my ($opt_vmid, $full) = @_;
2598
2599     my $res = {};
2600
2601     my $storecfg = PVE::Storage::config();
2602
2603     my $list = vzlist();
2604     my $defaults = load_defaults();
2605
2606     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2607
2608     my $cpucount = $cpuinfo->{cpus} || 1;
2609
2610     foreach my $vmid (keys %$list) {
2611         next if $opt_vmid && ($vmid ne $opt_vmid);
2612
2613         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2614         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2615
2616         my $d = {};
2617         $d->{pid} = $list->{$vmid}->{pid};
2618
2619         # fixme: better status?
2620         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2621
2622         my $size = disksize($storecfg, $conf);
2623         if (defined($size)) {
2624             $d->{disk} = 0; # no info available
2625             $d->{maxdisk} = $size;
2626         } else {
2627             $d->{disk} = 0;
2628             $d->{maxdisk} = 0;
2629         }
2630
2631         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2632             * ($conf->{cores} || $defaults->{cores});
2633         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2634         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2635
2636         $d->{name} = $conf->{name} || "VM $vmid";
2637         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2638             : $defaults->{memory}*(1024*1024);
2639
2640         if ($conf->{balloon}) {
2641             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2642             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2643                 : $defaults->{shares};
2644         }
2645
2646         $d->{uptime} = 0;
2647         $d->{cpu} = 0;
2648         $d->{mem} = 0;
2649
2650         $d->{netout} = 0;
2651         $d->{netin} = 0;
2652
2653         $d->{diskread} = 0;
2654         $d->{diskwrite} = 0;
2655
2656         $d->{template} = PVE::QemuConfig->is_template($conf);
2657
2658         $res->{$vmid} = $d;
2659     }
2660
2661     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2662     foreach my $dev (keys %$netdev) {
2663         next if $dev !~ m/^tap([1-9]\d*)i/;
2664         my $vmid = $1;
2665         my $d = $res->{$vmid};
2666         next if !$d;
2667
2668         $d->{netout} += $netdev->{$dev}->{receive};
2669         $d->{netin} += $netdev->{$dev}->{transmit};
2670
2671         if ($full) {
2672             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2673             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2674         }
2675
2676     }
2677
2678     my $ctime = gettimeofday;
2679
2680     foreach my $vmid (keys %$list) {
2681
2682         my $d = $res->{$vmid};
2683         my $pid = $d->{pid};
2684         next if !$pid;
2685
2686         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2687         next if !$pstat; # not running
2688
2689         my $used = $pstat->{utime} + $pstat->{stime};
2690
2691         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2692
2693         if ($pstat->{vsize}) {
2694             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2695         }
2696
2697         my $old = $last_proc_pid_stat->{$pid};
2698         if (!$old) {
2699             $last_proc_pid_stat->{$pid} = {
2700                 time => $ctime,
2701                 used => $used,
2702                 cpu => 0,
2703             };
2704             next;
2705         }
2706
2707         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2708
2709         if ($dtime > 1000) {
2710             my $dutime = $used -  $old->{used};
2711
2712             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2713             $last_proc_pid_stat->{$pid} = {
2714                 time => $ctime,
2715                 used => $used,
2716                 cpu => $d->{cpu},
2717             };
2718         } else {
2719             $d->{cpu} = $old->{cpu};
2720         }
2721     }
2722
2723     return $res if !$full;
2724
2725     my $qmpclient = PVE::QMPClient->new();
2726
2727     my $ballooncb = sub {
2728         my ($vmid, $resp) = @_;
2729
2730         my $info = $resp->{'return'};
2731         return if !$info->{max_mem};
2732
2733         my $d = $res->{$vmid};
2734
2735         # use memory assigned to VM
2736         $d->{maxmem} = $info->{max_mem};
2737         $d->{balloon} = $info->{actual};
2738
2739         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2740             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2741             $d->{freemem} = $info->{free_mem};
2742         }
2743
2744         $d->{ballooninfo} = $info;
2745     };
2746
2747     my $blockstatscb = sub {
2748         my ($vmid, $resp) = @_;
2749         my $data = $resp->{'return'} || [];
2750         my $totalrdbytes = 0;
2751         my $totalwrbytes = 0;
2752
2753         for my $blockstat (@$data) {
2754             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2755             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2756
2757             $blockstat->{device} =~ s/drive-//;
2758             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2759         }
2760         $res->{$vmid}->{diskread} = $totalrdbytes;
2761         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2762     };
2763
2764     my $statuscb = sub {
2765         my ($vmid, $resp) = @_;
2766
2767         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2768         # this fails if ballon driver is not loaded, so this must be
2769         # the last commnand (following command are aborted if this fails).
2770         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2771
2772         my $status = 'unknown';
2773         if (!defined($status = $resp->{'return'}->{status})) {
2774             warn "unable to get VM status\n";
2775             return;
2776         }
2777
2778         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2779     };
2780
2781     foreach my $vmid (keys %$list) {
2782         next if $opt_vmid && ($vmid ne $opt_vmid);
2783         next if !$res->{$vmid}->{pid}; # not running
2784         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2785     }
2786
2787     $qmpclient->queue_execute(undef, 2);
2788
2789     foreach my $vmid (keys %$list) {
2790         next if $opt_vmid && ($vmid ne $opt_vmid);
2791         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2792     }
2793
2794     return $res;
2795 }
2796
2797 sub foreach_drive {
2798     my ($conf, $func, @param) = @_;
2799
2800     foreach my $ds (valid_drive_names()) {
2801         next if !defined($conf->{$ds});
2802
2803         my $drive = parse_drive($ds, $conf->{$ds});
2804         next if !$drive;
2805
2806         &$func($ds, $drive, @param);
2807     }
2808 }
2809
2810 sub foreach_volid {
2811     my ($conf, $func, @param) = @_;
2812
2813     my $volhash = {};
2814
2815     my $test_volid = sub {
2816         my ($volid, $is_cdrom, $replicate, $snapname) = @_;
2817
2818         return if !$volid;
2819
2820         $volhash->{$volid}->{cdrom} //= 1;
2821         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2822
2823         $volhash->{$volid}->{replicate} //= 0;
2824         $volhash->{$volid}->{replicate} = 1 if $replicate;
2825
2826         $volhash->{$volid}->{referenced_in_config} //= 0;
2827         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
2828
2829         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
2830             if defined($snapname);
2831     };
2832
2833     foreach_drive($conf, sub {
2834         my ($ds, $drive) = @_;
2835         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, undef);
2836     });
2837
2838     foreach my $snapname (keys %{$conf->{snapshots}}) {
2839         my $snap = $conf->{snapshots}->{$snapname};
2840         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
2841         foreach_drive($snap, sub {
2842             my ($ds, $drive) = @_;
2843             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $snapname);
2844         });
2845     }
2846
2847     foreach my $volid (keys %$volhash) {
2848         &$func($volid, $volhash->{$volid}, @param);
2849     }
2850 }
2851
2852 sub vga_conf_has_spice {
2853     my ($vga) = @_;
2854
2855     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2856
2857     return $1 || 1;
2858 }
2859
2860 sub config_to_command {
2861     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2862
2863     my $cmd = [];
2864     my $globalFlags = [];
2865     my $machineFlags = [];
2866     my $rtcFlags = [];
2867     my $cpuFlags = [];
2868     my $devices = [];
2869     my $pciaddr = '';
2870     my $bridges = {};
2871     my $kvmver = kvm_user_version();
2872     my $vernum = 0; # unknown
2873     my $ostype = $conf->{ostype};
2874     my $winversion = windows_version($ostype);
2875     my $kvm = $conf->{kvm} // 1;
2876
2877     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
2878
2879     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2880         $vernum = $1*1000000+$2*1000;
2881     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2882         $vernum = $1*1000000+$2*1000+$3;
2883     }
2884
2885     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2886
2887     my $have_ovz = -f '/proc/vz/vestat';
2888
2889     my $q35 = machine_type_is_q35($conf);
2890     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2891     my $machine_type = $forcemachine || $conf->{machine};
2892     my $use_old_bios_files = undef;
2893     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2894
2895     my $cpuunits = defined($conf->{cpuunits}) ?
2896             $conf->{cpuunits} : $defaults->{cpuunits};
2897
2898     push @$cmd, '/usr/bin/kvm';
2899
2900     push @$cmd, '-id', $vmid;
2901
2902     my $use_virtio = 0;
2903
2904     my $qmpsocket = qmp_socket($vmid);
2905     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2906     push @$cmd, '-mon', "chardev=qmp,mode=control";
2907
2908
2909     push @$cmd, '-pidfile' , pidfile_name($vmid);
2910
2911     push @$cmd, '-daemonize';
2912
2913     if ($conf->{smbios1}) {
2914         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2915     }
2916
2917     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2918         die "uefi base image not found\n" if ! -f $OVMF_CODE;
2919
2920         my $path;
2921         my $format;
2922         if (my $efidisk = $conf->{efidisk0}) {
2923             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
2924             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2925             $format = $d->{format};
2926             if ($storeid) {
2927                 $path = PVE::Storage::path($storecfg, $d->{file});
2928                 if (!defined($format)) {
2929                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2930                     $format = qemu_img_format($scfg, $volname);
2931                 }
2932             } else {
2933                 $path = $d->{file};
2934                 die "efidisk format must be specified\n"
2935                     if !defined($format);
2936             }
2937         } else {
2938             warn "no efidisk configured! Using temporary efivars disk.\n";
2939             $path = "/tmp/$vmid-ovmf.fd";
2940             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
2941             $format = 'raw';
2942         }
2943
2944         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
2945         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
2946     }
2947
2948
2949     # add usb controllers
2950     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2951     push @$devices, @usbcontrollers if @usbcontrollers;
2952     my $vga = $conf->{vga};
2953
2954     my $qxlnum = vga_conf_has_spice($vga);
2955     $vga = 'qxl' if $qxlnum;
2956
2957     if (!$vga) {
2958         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
2959             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
2960         } else {
2961             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
2962         }
2963     }
2964
2965     # enable absolute mouse coordinates (needed by vnc)
2966     my $tablet;
2967     if (defined($conf->{tablet})) {
2968         $tablet = $conf->{tablet};
2969     } else {
2970         $tablet = $defaults->{tablet};
2971         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2972         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
2973     }
2974
2975     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
2976
2977     my $kvm_off = 0;
2978     my $gpu_passthrough;
2979
2980     # host pci devices
2981     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
2982         my $d = parse_hostpci($conf->{"hostpci$i"});
2983         next if !$d;
2984
2985         my $pcie = $d->{pcie};
2986         if($pcie){
2987             die "q35 machine model is not enabled" if !$q35;
2988             $pciaddr = print_pcie_addr("hostpci$i");
2989         }else{
2990             $pciaddr = print_pci_addr("hostpci$i", $bridges);
2991         }
2992
2993         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
2994         my $romfile = $d->{romfile};
2995
2996         my $xvga = '';
2997         if ($d->{'x-vga'}) {
2998             $xvga = ',x-vga=on';
2999             $kvm_off = 1;
3000             $vga = 'none';
3001             $gpu_passthrough = 1;
3002
3003             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3004                 $xvga = "";
3005             }
3006         }
3007         my $pcidevices = $d->{pciid};
3008         my $multifunction = 1 if @$pcidevices > 1;
3009
3010         my $j=0;
3011         foreach my $pcidevice (@$pcidevices) {
3012
3013             my $id = "hostpci$i";
3014             $id .= ".$j" if $multifunction;
3015             my $addr = $pciaddr;
3016             $addr .= ".$j" if $multifunction;
3017             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3018
3019             if($j == 0){
3020                 $devicestr .= "$rombar$xvga";
3021                 $devicestr .= ",multifunction=on" if $multifunction;
3022                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3023             }
3024
3025             push @$devices, '-device', $devicestr;
3026             $j++;
3027         }
3028     }
3029
3030     # usb devices
3031     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3032     push @$devices, @usbdevices if @usbdevices;
3033     # serial devices
3034     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3035         if (my $path = $conf->{"serial$i"}) {
3036             if ($path eq 'socket') {
3037                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3038                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3039                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3040             } else {
3041                 die "no such serial device\n" if ! -c $path;
3042                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3043                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3044             }
3045         }
3046     }
3047
3048     # parallel devices
3049     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3050         if (my $path = $conf->{"parallel$i"}) {
3051             die "no such parallel device\n" if ! -c $path;
3052             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3053             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3054             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3055         }
3056     }
3057
3058     my $vmname = $conf->{name} || "vm$vmid";
3059
3060     push @$cmd, '-name', $vmname;
3061
3062     my $sockets = 1;
3063     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3064     $sockets = $conf->{sockets} if  $conf->{sockets};
3065
3066     my $cores = $conf->{cores} || 1;
3067
3068     my $maxcpus = $sockets * $cores;
3069
3070     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3071
3072     my $allowed_vcpus = $cpuinfo->{cpus};
3073
3074     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3075         if ($allowed_vcpus < $maxcpus);
3076
3077     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3078
3079         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3080         for (my $i = 2; $i <= $vcpus; $i++)  {
3081             my $cpustr = print_cpu_device($conf,$i);
3082             push @$cmd, '-device', $cpustr;
3083         }
3084
3085     } else {
3086
3087         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3088     }
3089     push @$cmd, '-nodefaults';
3090
3091     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3092
3093     my $bootindex_hash = {};
3094     my $i = 1;
3095     foreach my $o (split(//, $bootorder)) {
3096         $bootindex_hash->{$o} = $i*100;
3097         $i++;
3098     }
3099
3100     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3101
3102     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3103
3104     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3105
3106     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3107
3108     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3109         my $socket = vnc_socket($vmid);
3110         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3111     } else {
3112         push @$cmd, '-nographic';
3113     }
3114
3115     # time drift fix
3116     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3117
3118     my $useLocaltime = $conf->{localtime};
3119
3120     if ($winversion >= 5) { # windows
3121         $useLocaltime = 1 if !defined($conf->{localtime});
3122
3123         # use time drift fix when acpi is enabled
3124         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3125             $tdf = 1 if !defined($conf->{tdf});
3126         }
3127     }
3128
3129     if ($winversion >= 6) {
3130         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3131         push @$cmd, '-no-hpet';
3132     }
3133
3134     push @$rtcFlags, 'driftfix=slew' if $tdf;
3135
3136     if (!$kvm) {
3137         push @$machineFlags, 'accel=tcg';
3138     }
3139
3140     if ($machine_type) {
3141         push @$machineFlags, "type=${machine_type}";
3142     }
3143
3144     if ($conf->{startdate}) {
3145         push @$rtcFlags, "base=$conf->{startdate}";
3146     } elsif ($useLocaltime) {
3147         push @$rtcFlags, 'base=localtime';
3148     }
3149
3150     my $cpu = $kvm ? "kvm64" : "qemu64";
3151     if (my $cputype = $conf->{cpu}) {
3152         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3153             or die "Cannot parse cpu description: $cputype\n";
3154         $cpu = $cpuconf->{cputype};
3155         $kvm_off = 1 if $cpuconf->{hidden};
3156
3157         if (defined(my $flags = $cpuconf->{flags})) {
3158             push @$cpuFlags, $flags;
3159         }
3160     }
3161
3162     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3163
3164     push @$cpuFlags , '-x2apic'
3165         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3166
3167     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3168
3169     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3170
3171     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3172
3173         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3174         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3175     }
3176
3177     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3178
3179     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3180
3181     push @$cpuFlags, 'kvm=off' if $kvm_off;
3182
3183     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3184         die "internal error"; # should not happen
3185
3186     push @$cpuFlags, "vendor=${cpu_vendor}"
3187         if $cpu_vendor ne 'default';
3188
3189     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3190
3191     push @$cmd, '-cpu', $cpu;
3192
3193     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3194
3195     push @$cmd, '-S' if $conf->{freeze};
3196
3197     # set keyboard layout
3198     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3199     push @$cmd, '-k', $kb if $kb;
3200
3201     # enable sound
3202     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3203     #push @$cmd, '-soundhw', 'es1370';
3204     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3205
3206     if($conf->{agent}) {
3207         my $qgasocket = qmp_socket($vmid, 1);
3208         my $pciaddr = print_pci_addr("qga0", $bridges);
3209         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3210         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3211         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3212     }
3213
3214     my $spice_port;
3215
3216     if ($qxlnum) {
3217         if ($qxlnum > 1) {
3218             if ($winversion){
3219                 for(my $i = 1; $i < $qxlnum; $i++){
3220                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3221                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3222                 }
3223             } else {
3224                 # assume other OS works like Linux
3225                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3226                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3227             }
3228         }
3229
3230         my $pciaddr = print_pci_addr("spice", $bridges);
3231
3232         my $nodename = PVE::INotify::nodename();
3233         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3234         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3235         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3236         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3237         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3238
3239         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3240
3241         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3242         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3243         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3244     }
3245
3246     # enable balloon by default, unless explicitly disabled
3247     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3248         $pciaddr = print_pci_addr("balloon0", $bridges);
3249         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3250     }
3251
3252     if ($conf->{watchdog}) {
3253         my $wdopts = parse_watchdog($conf->{watchdog});
3254         $pciaddr = print_pci_addr("watchdog", $bridges);
3255         my $watchdog = $wdopts->{model} || 'i6300esb';
3256         push @$devices, '-device', "$watchdog$pciaddr";
3257         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3258     }
3259
3260     my $vollist = [];
3261     my $scsicontroller = {};
3262     my $ahcicontroller = {};
3263     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3264
3265     # Add iscsi initiator name if available
3266     if (my $initiator = get_initiator_name()) {
3267         push @$devices, '-iscsi', "initiator-name=$initiator";
3268     }
3269
3270     foreach_drive($conf, sub {
3271         my ($ds, $drive) = @_;
3272
3273         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3274             push @$vollist, $drive->{file};
3275         }
3276
3277         # ignore efidisk here, already added in bios/fw handling code above
3278         return if $drive->{interface} eq 'efidisk';
3279
3280         $use_virtio = 1 if $ds =~ m/^virtio/;
3281
3282         if (drive_is_cdrom ($drive)) {
3283             if ($bootindex_hash->{d}) {
3284                 $drive->{bootindex} = $bootindex_hash->{d};
3285                 $bootindex_hash->{d} += 1;
3286             }
3287         } else {
3288             if ($bootindex_hash->{c}) {
3289                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3290                 $bootindex_hash->{c} += 1;
3291             }
3292         }
3293
3294         if($drive->{interface} eq 'virtio'){
3295            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3296         }
3297
3298         if ($drive->{interface} eq 'scsi') {
3299
3300             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3301
3302             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3303             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3304
3305             my $iothread = '';
3306             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3307                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3308                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3309             } elsif ($drive->{iothread}) {
3310                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3311             }
3312
3313             my $queues = '';
3314             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3315                 $queues = ",num_queues=$drive->{queues}";
3316             }
3317
3318             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3319             $scsicontroller->{$controller}=1;
3320         }
3321
3322         if ($drive->{interface} eq 'sata') {
3323            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3324            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3325            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3326            $ahcicontroller->{$controller}=1;
3327         }
3328
3329         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3330         push @$devices, '-drive',$drive_cmd;
3331         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3332     });
3333
3334     for (my $i = 0; $i < $MAX_NETS; $i++) {
3335          next if !$conf->{"net$i"};
3336          my $d = parse_net($conf->{"net$i"});
3337          next if !$d;
3338
3339          $use_virtio = 1 if $d->{model} eq 'virtio';
3340
3341          if ($bootindex_hash->{n}) {
3342             $d->{bootindex} = $bootindex_hash->{n};
3343             $bootindex_hash->{n} += 1;
3344          }
3345
3346          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3347          push @$devices, '-netdev', $netdevfull;
3348
3349          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3350          push @$devices, '-device', $netdevicefull;
3351     }
3352
3353     if (!$q35) {
3354         # add pci bridges
3355         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3356            $bridges->{1} = 1;
3357            $bridges->{2} = 1;
3358         }
3359
3360         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3361
3362         while (my ($k, $v) = each %$bridges) {
3363             $pciaddr = print_pci_addr("pci.$k");
3364             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3365         }
3366     }
3367
3368     # add custom args
3369     if ($conf->{args}) {
3370         my $aa = PVE::Tools::split_args($conf->{args});
3371         push @$cmd, @$aa;
3372     }
3373
3374     push @$cmd, @$devices;
3375     push @$cmd, '-rtc', join(',', @$rtcFlags)
3376         if scalar(@$rtcFlags);
3377     push @$cmd, '-machine', join(',', @$machineFlags)
3378         if scalar(@$machineFlags);
3379     push @$cmd, '-global', join(',', @$globalFlags)
3380         if scalar(@$globalFlags);
3381
3382     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3383 }
3384
3385 sub vnc_socket {
3386     my ($vmid) = @_;
3387     return "${var_run_tmpdir}/$vmid.vnc";
3388 }
3389
3390 sub spice_port {
3391     my ($vmid) = @_;
3392
3393     my $res = vm_mon_cmd($vmid, 'query-spice');
3394
3395     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3396 }
3397
3398 sub qmp_socket {
3399     my ($vmid, $qga) = @_;
3400     my $sockettype = $qga ? 'qga' : 'qmp';
3401     return "${var_run_tmpdir}/$vmid.$sockettype";
3402 }
3403
3404 sub pidfile_name {
3405     my ($vmid) = @_;
3406     return "${var_run_tmpdir}/$vmid.pid";
3407 }
3408
3409 sub vm_devices_list {
3410     my ($vmid) = @_;
3411
3412     my $res = vm_mon_cmd($vmid, 'query-pci');
3413     my $devices = {};
3414     foreach my $pcibus (@$res) {
3415         foreach my $device (@{$pcibus->{devices}}) {
3416             next if !$device->{'qdev_id'};
3417             if ($device->{'pci_bridge'}) {
3418                 $devices->{$device->{'qdev_id'}} = 1;
3419                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3420                     next if !$bridge_device->{'qdev_id'};
3421                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3422                     $devices->{$device->{'qdev_id'}}++;
3423                 }
3424             } else {
3425                 $devices->{$device->{'qdev_id'}} = 1;
3426             }
3427         }
3428     }
3429
3430     my $resblock = vm_mon_cmd($vmid, 'query-block');
3431     foreach my $block (@$resblock) {
3432         if($block->{device} =~ m/^drive-(\S+)/){
3433                 $devices->{$1} = 1;
3434         }
3435     }
3436
3437     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3438     foreach my $mice (@$resmice) {
3439         if ($mice->{name} eq 'QEMU HID Tablet') {
3440             $devices->{tablet} = 1;
3441             last;
3442         }
3443     }
3444
3445     # for usb devices there is no query-usb
3446     # but we can iterate over the entries in
3447     # qom-list path=/machine/peripheral
3448     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3449     foreach my $per (@$resperipheral) {
3450         if ($per->{name} =~ m/^usb\d+$/) {
3451             $devices->{$per->{name}} = 1;
3452         }
3453     }
3454
3455     return $devices;
3456 }
3457
3458 sub vm_deviceplug {
3459     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3460
3461     my $q35 = machine_type_is_q35($conf);
3462
3463     my $devices_list = vm_devices_list($vmid);
3464     return 1 if defined($devices_list->{$deviceid});
3465
3466     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3467
3468     if ($deviceid eq 'tablet') {
3469
3470         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3471
3472     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3473
3474         die "usb hotplug currently not reliable\n";
3475         # since we can't reliably hot unplug all added usb devices
3476         # and usb passthrough disables live migration
3477         # we disable usb hotplugging for now
3478         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3479
3480     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3481
3482         qemu_iothread_add($vmid, $deviceid, $device);
3483
3484         qemu_driveadd($storecfg, $vmid, $device);
3485         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3486
3487         qemu_deviceadd($vmid, $devicefull);
3488         eval { qemu_deviceaddverify($vmid, $deviceid); };
3489         if (my $err = $@) {
3490             eval { qemu_drivedel($vmid, $deviceid); };
3491             warn $@ if $@;
3492             die $err;
3493         }
3494
3495     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3496
3497
3498         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3499         my $pciaddr = print_pci_addr($deviceid);
3500         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3501
3502         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3503
3504         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3505             qemu_iothread_add($vmid, $deviceid, $device);
3506             $devicefull .= ",iothread=iothread-$deviceid";
3507         }
3508
3509         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3510             $devicefull .= ",num_queues=$device->{queues}";
3511         }
3512
3513         qemu_deviceadd($vmid, $devicefull);
3514         qemu_deviceaddverify($vmid, $deviceid);
3515
3516     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3517
3518         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3519         qemu_driveadd($storecfg, $vmid, $device);
3520
3521         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3522         eval { qemu_deviceadd($vmid, $devicefull); };
3523         if (my $err = $@) {
3524             eval { qemu_drivedel($vmid, $deviceid); };
3525             warn $@ if $@;
3526             die $err;
3527         }
3528
3529     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3530
3531         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3532
3533         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3534         my $use_old_bios_files = undef;
3535         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3536
3537         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3538         qemu_deviceadd($vmid, $netdevicefull);
3539         eval { qemu_deviceaddverify($vmid, $deviceid); };
3540         if (my $err = $@) {
3541             eval { qemu_netdevdel($vmid, $deviceid); };
3542             warn $@ if $@;
3543             die $err;
3544         }
3545
3546     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3547
3548         my $bridgeid = $2;
3549         my $pciaddr = print_pci_addr($deviceid);
3550         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3551
3552         qemu_deviceadd($vmid, $devicefull);
3553         qemu_deviceaddverify($vmid, $deviceid);
3554
3555     } else {
3556         die "can't hotplug device '$deviceid'\n";
3557     }
3558
3559     return 1;
3560 }
3561
3562 # fixme: this should raise exceptions on error!
3563 sub vm_deviceunplug {
3564     my ($vmid, $conf, $deviceid) = @_;
3565
3566     my $devices_list = vm_devices_list($vmid);
3567     return 1 if !defined($devices_list->{$deviceid});
3568
3569     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3570
3571     if ($deviceid eq 'tablet') {
3572
3573         qemu_devicedel($vmid, $deviceid);
3574
3575     } elsif ($deviceid =~ m/^usb\d+$/) {
3576
3577         die "usb hotplug currently not reliable\n";
3578         # when unplugging usb devices this way,
3579         # there may be remaining usb controllers/hubs
3580         # so we disable it for now
3581         qemu_devicedel($vmid, $deviceid);
3582         qemu_devicedelverify($vmid, $deviceid);
3583
3584     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3585
3586         qemu_devicedel($vmid, $deviceid);
3587         qemu_devicedelverify($vmid, $deviceid);
3588         qemu_drivedel($vmid, $deviceid);
3589         qemu_iothread_del($conf, $vmid, $deviceid);
3590
3591     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3592
3593         qemu_devicedel($vmid, $deviceid);
3594         qemu_devicedelverify($vmid, $deviceid);
3595         qemu_iothread_del($conf, $vmid, $deviceid);
3596
3597     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3598
3599         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3600         my $device = parse_drive($deviceid, $conf->{$deviceid});
3601         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3602
3603         qemu_devicedel($vmid, $deviceid);
3604         qemu_drivedel($vmid, $deviceid);
3605         qemu_deletescsihw($conf, $vmid, $deviceid);
3606
3607     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3608
3609         qemu_devicedel($vmid, $deviceid);
3610         qemu_devicedelverify($vmid, $deviceid);
3611         qemu_netdevdel($vmid, $deviceid);
3612
3613     } else {
3614         die "can't unplug device '$deviceid'\n";
3615     }
3616
3617     return 1;
3618 }
3619
3620 sub qemu_deviceadd {
3621     my ($vmid, $devicefull) = @_;
3622
3623     $devicefull = "driver=".$devicefull;
3624     my %options =  split(/[=,]/, $devicefull);
3625
3626     vm_mon_cmd($vmid, "device_add" , %options);
3627 }
3628
3629 sub qemu_devicedel {
3630     my ($vmid, $deviceid) = @_;
3631
3632     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3633 }
3634
3635 sub qemu_iothread_add {
3636     my($vmid, $deviceid, $device) = @_;
3637
3638     if ($device->{iothread}) {
3639         my $iothreads = vm_iothreads_list($vmid);
3640         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3641     }
3642 }
3643
3644 sub qemu_iothread_del {
3645     my($conf, $vmid, $deviceid) = @_;
3646
3647     my $device = parse_drive($deviceid, $conf->{$deviceid});
3648     if ($device->{iothread}) {
3649         my $iothreads = vm_iothreads_list($vmid);
3650         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3651     }
3652 }
3653
3654 sub qemu_objectadd {
3655     my($vmid, $objectid, $qomtype) = @_;
3656
3657     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3658
3659     return 1;
3660 }
3661
3662 sub qemu_objectdel {
3663     my($vmid, $objectid) = @_;
3664
3665     vm_mon_cmd($vmid, "object-del", id => $objectid);
3666
3667     return 1;
3668 }
3669
3670 sub qemu_driveadd {
3671     my ($storecfg, $vmid, $device) = @_;
3672
3673     my $drive = print_drive_full($storecfg, $vmid, $device);
3674     $drive =~ s/\\/\\\\/g;
3675     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3676
3677     # If the command succeeds qemu prints: "OK"
3678     return 1 if $ret =~ m/OK/s;
3679
3680     die "adding drive failed: $ret\n";
3681 }
3682
3683 sub qemu_drivedel {
3684     my($vmid, $deviceid) = @_;
3685
3686     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3687     $ret =~ s/^\s+//;
3688
3689     return 1 if $ret eq "";
3690
3691     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3692     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3693
3694     die "deleting drive $deviceid failed : $ret\n";
3695 }
3696
3697 sub qemu_deviceaddverify {
3698     my ($vmid, $deviceid) = @_;
3699
3700     for (my $i = 0; $i <= 5; $i++) {
3701          my $devices_list = vm_devices_list($vmid);
3702          return 1 if defined($devices_list->{$deviceid});
3703          sleep 1;
3704     }
3705
3706     die "error on hotplug device '$deviceid'\n";
3707 }
3708
3709
3710 sub qemu_devicedelverify {
3711     my ($vmid, $deviceid) = @_;
3712
3713     # need to verify that the device is correctly removed as device_del
3714     # is async and empty return is not reliable
3715
3716     for (my $i = 0; $i <= 5; $i++) {
3717          my $devices_list = vm_devices_list($vmid);
3718          return 1 if !defined($devices_list->{$deviceid});
3719          sleep 1;
3720     }
3721
3722     die "error on hot-unplugging device '$deviceid'\n";
3723 }
3724
3725 sub qemu_findorcreatescsihw {
3726     my ($storecfg, $conf, $vmid, $device) = @_;
3727
3728     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3729
3730     my $scsihwid="$controller_prefix$controller";
3731     my $devices_list = vm_devices_list($vmid);
3732
3733     if(!defined($devices_list->{$scsihwid})) {
3734         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3735     }
3736
3737     return 1;
3738 }
3739
3740 sub qemu_deletescsihw {
3741     my ($conf, $vmid, $opt) = @_;
3742
3743     my $device = parse_drive($opt, $conf->{$opt});
3744
3745     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3746         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3747         return 1;
3748     }
3749
3750     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3751
3752     my $devices_list = vm_devices_list($vmid);
3753     foreach my $opt (keys %{$devices_list}) {
3754         if (PVE::QemuServer::is_valid_drivename($opt)) {
3755             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3756             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3757                 return 1;
3758             }
3759         }
3760     }
3761
3762     my $scsihwid="scsihw$controller";
3763
3764     vm_deviceunplug($vmid, $conf, $scsihwid);
3765
3766     return 1;
3767 }
3768
3769 sub qemu_add_pci_bridge {
3770     my ($storecfg, $conf, $vmid, $device) = @_;
3771
3772     my $bridges = {};
3773
3774     my $bridgeid;
3775
3776     print_pci_addr($device, $bridges);
3777
3778     while (my ($k, $v) = each %$bridges) {
3779         $bridgeid = $k;
3780     }
3781     return 1 if !defined($bridgeid) || $bridgeid < 1;
3782
3783     my $bridge = "pci.$bridgeid";
3784     my $devices_list = vm_devices_list($vmid);
3785
3786     if (!defined($devices_list->{$bridge})) {
3787         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3788     }
3789
3790     return 1;
3791 }
3792
3793 sub qemu_set_link_status {
3794     my ($vmid, $device, $up) = @_;
3795
3796     vm_mon_cmd($vmid, "set_link", name => $device,
3797                up => $up ? JSON::true : JSON::false);
3798 }
3799
3800 sub qemu_netdevadd {
3801     my ($vmid, $conf, $device, $deviceid) = @_;
3802
3803     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3804     my %options =  split(/[=,]/, $netdev);
3805
3806     vm_mon_cmd($vmid, "netdev_add",  %options);
3807     return 1;
3808 }
3809
3810 sub qemu_netdevdel {
3811     my ($vmid, $deviceid) = @_;
3812
3813     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3814 }
3815
3816 sub qemu_usb_hotplug {
3817     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3818
3819     return if !$device;
3820
3821     # remove the old one first
3822     vm_deviceunplug($vmid, $conf, $deviceid);
3823
3824     # check if xhci controller is necessary and available
3825     if ($device->{usb3}) {
3826
3827         my $devicelist = vm_devices_list($vmid);
3828
3829         if (!$devicelist->{xhci}) {
3830             my $pciaddr = print_pci_addr("xhci");
3831             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3832         }
3833     }
3834     my $d = parse_usb_device($device->{host});
3835     $d->{usb3} = $device->{usb3};
3836
3837     # add the new one
3838     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3839 }
3840
3841 sub qemu_cpu_hotplug {
3842     my ($vmid, $conf, $vcpus) = @_;
3843
3844     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3845
3846     my $sockets = 1;
3847     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3848     $sockets = $conf->{sockets} if  $conf->{sockets};
3849     my $cores = $conf->{cores} || 1;
3850     my $maxcpus = $sockets * $cores;
3851
3852     $vcpus = $maxcpus if !$vcpus;
3853
3854     die "you can't add more vcpus than maxcpus\n"
3855         if $vcpus > $maxcpus;
3856
3857     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3858
3859     if ($vcpus < $currentvcpus) {
3860
3861         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3862
3863             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3864                 qemu_devicedel($vmid, "cpu$i");
3865                 my $retry = 0;
3866                 my $currentrunningvcpus = undef;
3867                 while (1) {
3868                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3869                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3870                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3871                     $retry++;
3872                     sleep 1;
3873                 }
3874                 #update conf after each succesfull cpu unplug
3875                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3876                 PVE::QemuConfig->write_config($vmid, $conf);
3877             }
3878         } else {
3879             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3880         }
3881
3882         return;
3883     }
3884
3885     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3886     die "vcpus in running vm does not match its configuration\n"
3887         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3888
3889     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3890
3891         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3892             my $cpustr = print_cpu_device($conf, $i);
3893             qemu_deviceadd($vmid, $cpustr);
3894
3895             my $retry = 0;
3896             my $currentrunningvcpus = undef;
3897             while (1) {
3898                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3899                 last if scalar(@{$currentrunningvcpus}) == $i;
3900                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3901                 sleep 1;
3902                 $retry++;
3903             }
3904             #update conf after each succesfull cpu hotplug
3905             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3906             PVE::QemuConfig->write_config($vmid, $conf);
3907         }
3908     } else {
3909
3910         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3911             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3912         }
3913     }
3914 }
3915
3916 sub qemu_block_set_io_throttle {
3917     my ($vmid, $deviceid,
3918         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3919         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3920         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3921         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3922
3923     return if !check_running($vmid) ;
3924
3925     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3926         bps => int($bps),
3927         bps_rd => int($bps_rd),
3928         bps_wr => int($bps_wr),
3929         iops => int($iops),
3930         iops_rd => int($iops_rd),
3931         iops_wr => int($iops_wr),
3932         bps_max => int($bps_max),
3933         bps_rd_max => int($bps_rd_max),
3934         bps_wr_max => int($bps_wr_max),
3935         iops_max => int($iops_max),
3936         iops_rd_max => int($iops_rd_max),
3937         iops_wr_max => int($iops_wr_max),
3938         bps_max_length => int($bps_max_length),
3939         bps_rd_max_length => int($bps_rd_max_length),
3940         bps_wr_max_length => int($bps_wr_max_length),
3941         iops_max_length => int($iops_max_length),
3942         iops_rd_max_length => int($iops_rd_max_length),
3943         iops_wr_max_length => int($iops_wr_max_length),
3944     );
3945
3946 }
3947
3948 # old code, only used to shutdown old VM after update
3949 sub __read_avail {
3950     my ($fh, $timeout) = @_;
3951
3952     my $sel = new IO::Select;
3953     $sel->add($fh);
3954
3955     my $res = '';
3956     my $buf;
3957
3958     my @ready;
3959     while (scalar (@ready = $sel->can_read($timeout))) {
3960         my $count;
3961         if ($count = $fh->sysread($buf, 8192)) {
3962             if ($buf =~ /^(.*)\(qemu\) $/s) {
3963                 $res .= $1;
3964                 last;
3965             } else {
3966                 $res .= $buf;
3967             }
3968         } else {
3969             if (!defined($count)) {
3970                 die "$!\n";
3971             }
3972             last;
3973         }
3974     }
3975
3976     die "monitor read timeout\n" if !scalar(@ready);
3977
3978     return $res;
3979 }
3980
3981 # old code, only used to shutdown old VM after update
3982 sub vm_monitor_command {
3983     my ($vmid, $cmdstr, $nocheck) = @_;
3984
3985     my $res;
3986
3987     eval {
3988         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
3989
3990         my $sname = "${var_run_tmpdir}/$vmid.mon";
3991
3992         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
3993             die "unable to connect to VM $vmid socket - $!\n";
3994
3995         my $timeout = 3;
3996
3997         # hack: migrate sometime blocks the monitor (when migrate_downtime
3998         # is set)
3999         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4000             $timeout = 60*60; # 1 hour
4001         }
4002
4003         # read banner;
4004         my $data = __read_avail($sock, $timeout);
4005
4006         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
4007             die "got unexpected qemu monitor banner\n";
4008         }
4009
4010         my $sel = new IO::Select;
4011         $sel->add($sock);
4012
4013         if (!scalar(my @ready = $sel->can_write($timeout))) {
4014             die "monitor write error - timeout";
4015         }
4016
4017         my $fullcmd = "$cmdstr\r";
4018
4019         # syslog('info', "VM $vmid monitor command: $cmdstr");
4020
4021         my $b;
4022         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
4023             die "monitor write error - $!";
4024         }
4025
4026         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4027
4028         $timeout = 20;
4029
4030         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4031             $timeout = 60*60; # 1 hour
4032         } elsif ($cmdstr =~ m/^(eject|change)/) {
4033             $timeout = 60; # note: cdrom mount command is slow
4034         }
4035         if ($res = __read_avail($sock, $timeout)) {
4036
4037             my @lines = split("\r?\n", $res);
4038
4039             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4040
4041             $res = join("\n", @lines);
4042             $res .= "\n";
4043         }
4044     };
4045
4046     my $err = $@;
4047
4048     if ($err) {
4049         syslog("err", "VM $vmid monitor command failed - $err");
4050         die $err;
4051     }
4052
4053     return $res;
4054 }
4055
4056 sub qemu_block_resize {
4057     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4058
4059     my $running = check_running($vmid);
4060
4061     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4062
4063     return if !$running;
4064
4065     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4066
4067 }
4068
4069 sub qemu_volume_snapshot {
4070     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4071
4072     my $running = check_running($vmid);
4073
4074     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4075         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4076     } else {
4077         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4078     }
4079 }
4080
4081 sub qemu_volume_snapshot_delete {
4082     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4083
4084     my $running = check_running($vmid);
4085
4086     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4087         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4088     } else {
4089         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4090     }
4091 }
4092
4093 sub set_migration_caps {
4094     my ($vmid) = @_;
4095
4096     my $cap_ref = [];
4097
4098     my $enabled_cap = {
4099         "auto-converge" => 1,
4100         "xbzrle" => 1,
4101         "x-rdma-pin-all" => 0,
4102         "zero-blocks" => 0,
4103         "compress" => 0
4104     };
4105
4106     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4107
4108     for my $supported_capability (@$supported_capabilities) {
4109         push @$cap_ref, {
4110             capability => $supported_capability->{capability},
4111             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4112         };
4113     }
4114
4115     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4116 }
4117
4118 my $fast_plug_option = {
4119     'lock' => 1,
4120     'name' => 1,
4121     'onboot' => 1,
4122     'shares' => 1,
4123     'startup' => 1,
4124     'description' => 1,
4125     'protection' => 1,
4126     'vmstatestorage' => 1,
4127 };
4128
4129 # hotplug changes in [PENDING]
4130 # $selection hash can be used to only apply specified options, for
4131 # example: { cores => 1 } (only apply changed 'cores')
4132 # $errors ref is used to return error messages
4133 sub vmconfig_hotplug_pending {
4134     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4135
4136     my $defaults = load_defaults();
4137
4138     # commit values which do not have any impact on running VM first
4139     # Note: those option cannot raise errors, we we do not care about
4140     # $selection and always apply them.
4141
4142     my $add_error = sub {
4143         my ($opt, $msg) = @_;
4144         $errors->{$opt} = "hotplug problem - $msg";
4145     };
4146
4147     my $changes = 0;
4148     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4149         if ($fast_plug_option->{$opt}) {
4150             $conf->{$opt} = $conf->{pending}->{$opt};
4151             delete $conf->{pending}->{$opt};
4152             $changes = 1;
4153         }
4154     }
4155
4156     if ($changes) {
4157         PVE::QemuConfig->write_config($vmid, $conf);
4158         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4159     }
4160
4161     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4162
4163     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4164     while (my ($opt, $force) = each %$pending_delete_hash) {
4165         next if $selection && !$selection->{$opt};
4166         eval {
4167             if ($opt eq 'hotplug') {
4168                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4169             } elsif ($opt eq 'tablet') {
4170                 die "skip\n" if !$hotplug_features->{usb};
4171                 if ($defaults->{tablet}) {
4172                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4173                 } else {
4174                     vm_deviceunplug($vmid, $conf, $opt);
4175                 }
4176             } elsif ($opt =~ m/^usb\d+/) {
4177                 die "skip\n";
4178                 # since we cannot reliably hot unplug usb devices
4179                 # we are disabling it
4180                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4181                 vm_deviceunplug($vmid, $conf, $opt);
4182             } elsif ($opt eq 'vcpus') {
4183                 die "skip\n" if !$hotplug_features->{cpu};
4184                 qemu_cpu_hotplug($vmid, $conf, undef);
4185             } elsif ($opt eq 'balloon') {
4186                 # enable balloon device is not hotpluggable
4187                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4188             } elsif ($fast_plug_option->{$opt}) {
4189                 # do nothing
4190             } elsif ($opt =~ m/^net(\d+)$/) {
4191                 die "skip\n" if !$hotplug_features->{network};
4192                 vm_deviceunplug($vmid, $conf, $opt);
4193             } elsif (is_valid_drivename($opt)) {
4194                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4195                 vm_deviceunplug($vmid, $conf, $opt);
4196                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4197             } elsif ($opt =~ m/^memory$/) {
4198                 die "skip\n" if !$hotplug_features->{memory};
4199                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4200             } elsif ($opt eq 'cpuunits') {
4201                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4202             } elsif ($opt eq 'cpulimit') {
4203                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4204             } else {
4205                 die "skip\n";
4206             }
4207         };
4208         if (my $err = $@) {
4209             &$add_error($opt, $err) if $err ne "skip\n";
4210         } else {
4211             # save new config if hotplug was successful
4212             delete $conf->{$opt};
4213             vmconfig_undelete_pending_option($conf, $opt);
4214             PVE::QemuConfig->write_config($vmid, $conf);
4215             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4216         }
4217     }
4218
4219     foreach my $opt (keys %{$conf->{pending}}) {
4220         next if $selection && !$selection->{$opt};
4221         my $value = $conf->{pending}->{$opt};
4222         eval {
4223             if ($opt eq 'hotplug') {
4224                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4225             } elsif ($opt eq 'tablet') {
4226                 die "skip\n" if !$hotplug_features->{usb};
4227                 if ($value == 1) {
4228                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4229                 } elsif ($value == 0) {
4230                     vm_deviceunplug($vmid, $conf, $opt);
4231                 }
4232             } elsif ($opt =~ m/^usb\d+$/) {
4233                 die "skip\n";
4234                 # since we cannot reliably hot unplug usb devices
4235                 # we are disabling it
4236                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4237                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4238                 die "skip\n" if !$d;
4239                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4240             } elsif ($opt eq 'vcpus') {
4241                 die "skip\n" if !$hotplug_features->{cpu};
4242                 qemu_cpu_hotplug($vmid, $conf, $value);
4243             } elsif ($opt eq 'balloon') {
4244                 # enable/disable balloning device is not hotpluggable
4245                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4246                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4247                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4248
4249                 # allow manual ballooning if shares is set to zero
4250                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4251                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4252                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4253                 }
4254             } elsif ($opt =~ m/^net(\d+)$/) {
4255                 # some changes can be done without hotplug
4256                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4257                                     $vmid, $opt, $value);
4258             } elsif (is_valid_drivename($opt)) {
4259                 # some changes can be done without hotplug
4260                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4261                                      $vmid, $opt, $value, 1);
4262             } elsif ($opt =~ m/^memory$/) { #dimms
4263                 die "skip\n" if !$hotplug_features->{memory};
4264                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4265             } elsif ($opt eq 'cpuunits') {
4266                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4267             } elsif ($opt eq 'cpulimit') {
4268                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4269                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4270             } else {
4271                 die "skip\n";  # skip non-hot-pluggable options
4272             }
4273         };
4274         if (my $err = $@) {
4275             &$add_error($opt, $err) if $err ne "skip\n";
4276         } else {
4277             # save new config if hotplug was successful
4278             $conf->{$opt} = $value;
4279             delete $conf->{pending}->{$opt};
4280             PVE::QemuConfig->write_config($vmid, $conf);
4281             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4282         }
4283     }
4284 }
4285
4286 sub try_deallocate_drive {
4287     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4288
4289     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4290         my $volid = $drive->{file};
4291         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4292             my $sid = PVE::Storage::parse_volume_id($volid);
4293             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4294
4295             # check if the disk is really unused
4296             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4297                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4298             PVE::Storage::vdisk_free($storecfg, $volid);
4299             return 1;
4300         } else {
4301             # If vm is not owner of this disk remove from config
4302             return 1;
4303         }
4304     }
4305
4306     return undef;
4307 }
4308
4309 sub vmconfig_delete_or_detach_drive {
4310     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4311
4312     my $drive = parse_drive($opt, $conf->{$opt});
4313
4314     my $rpcenv = PVE::RPCEnvironment::get();
4315     my $authuser = $rpcenv->get_user();
4316
4317     if ($force) {
4318         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4319         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4320     } else {
4321         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4322     }
4323 }
4324
4325 sub vmconfig_apply_pending {
4326     my ($vmid, $conf, $storecfg) = @_;
4327
4328     # cold plug
4329
4330     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4331     while (my ($opt, $force) = each %$pending_delete_hash) {
4332         die "internal error" if $opt =~ m/^unused/;
4333         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4334         if (!defined($conf->{$opt})) {
4335             vmconfig_undelete_pending_option($conf, $opt);
4336             PVE::QemuConfig->write_config($vmid, $conf);
4337         } elsif (is_valid_drivename($opt)) {
4338             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4339             vmconfig_undelete_pending_option($conf, $opt);
4340             delete $conf->{$opt};
4341             PVE::QemuConfig->write_config($vmid, $conf);
4342         } else {
4343             vmconfig_undelete_pending_option($conf, $opt);
4344             delete $conf->{$opt};
4345             PVE::QemuConfig->write_config($vmid, $conf);
4346         }
4347     }
4348
4349     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4350
4351     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4352         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4353
4354         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4355             # skip if nothing changed
4356         } elsif (is_valid_drivename($opt)) {
4357             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4358                 if defined($conf->{$opt});
4359             $conf->{$opt} = $conf->{pending}->{$opt};
4360         } else {
4361             $conf->{$opt} = $conf->{pending}->{$opt};
4362         }
4363
4364         delete $conf->{pending}->{$opt};
4365         PVE::QemuConfig->write_config($vmid, $conf);
4366     }
4367 }
4368
4369 my $safe_num_ne = sub {
4370     my ($a, $b) = @_;
4371
4372     return 0 if !defined($a) && !defined($b);
4373     return 1 if !defined($a);
4374     return 1 if !defined($b);
4375
4376     return $a != $b;
4377 };
4378
4379 my $safe_string_ne = sub {
4380     my ($a, $b) = @_;
4381
4382     return 0 if !defined($a) && !defined($b);
4383     return 1 if !defined($a);
4384     return 1 if !defined($b);
4385
4386     return $a ne $b;
4387 };
4388
4389 sub vmconfig_update_net {
4390     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4391
4392     my $newnet = parse_net($value);
4393
4394     if ($conf->{$opt}) {
4395         my $oldnet = parse_net($conf->{$opt});
4396
4397         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4398             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4399             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4400             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4401
4402             # for non online change, we try to hot-unplug
4403             die "skip\n" if !$hotplug;
4404             vm_deviceunplug($vmid, $conf, $opt);
4405         } else {
4406
4407             die "internal error" if $opt !~ m/net(\d+)/;
4408             my $iface = "tap${vmid}i$1";
4409
4410             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4411                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4412                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4413                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4414                 PVE::Network::tap_unplug($iface);
4415                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4416             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4417                 # Rate can be applied on its own but any change above needs to
4418                 # include the rate in tap_plug since OVS resets everything.
4419                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4420             }
4421
4422             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4423                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4424             }
4425
4426             return 1;
4427         }
4428     }
4429
4430     if ($hotplug) {
4431         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4432     } else {
4433         die "skip\n";
4434     }
4435 }
4436
4437 sub vmconfig_update_disk {
4438     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4439
4440     # fixme: do we need force?
4441
4442     my $drive = parse_drive($opt, $value);
4443
4444     if ($conf->{$opt}) {
4445
4446         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4447
4448             my $media = $drive->{media} || 'disk';
4449             my $oldmedia = $old_drive->{media} || 'disk';
4450             die "unable to change media type\n" if $media ne $oldmedia;
4451
4452             if (!drive_is_cdrom($old_drive)) {
4453
4454                 if ($drive->{file} ne $old_drive->{file}) {
4455
4456                     die "skip\n" if !$hotplug;
4457
4458                     # unplug and register as unused
4459                     vm_deviceunplug($vmid, $conf, $opt);
4460                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4461
4462                 } else {
4463                     # update existing disk
4464
4465                     # skip non hotpluggable value
4466                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4467                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4468                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4469                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4470                         die "skip\n";
4471                     }
4472
4473                     # apply throttle
4474                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4475                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4476                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4477                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4478                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4479                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4480                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4481                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4482                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4483                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4484                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4485                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4486                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4487                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4488                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4489                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4490                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4491                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4492
4493                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4494                                                    ($drive->{mbps} || 0)*1024*1024,
4495                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4496                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4497                                                    $drive->{iops} || 0,
4498                                                    $drive->{iops_rd} || 0,
4499                                                    $drive->{iops_wr} || 0,
4500                                                    ($drive->{mbps_max} || 0)*1024*1024,
4501                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4502                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4503                                                    $drive->{iops_max} || 0,
4504                                                    $drive->{iops_rd_max} || 0,
4505                                                    $drive->{iops_wr_max} || 0,
4506                                                    $drive->{bps_max_length} || 1,
4507                                                    $drive->{bps_rd_max_length} || 1,
4508                                                    $drive->{bps_wr_max_length} || 1,
4509                                                    $drive->{iops_max_length} || 1,
4510                                                    $drive->{iops_rd_max_length} || 1,
4511                                                    $drive->{iops_wr_max_length} || 1);
4512
4513                     }
4514
4515                     return 1;
4516                 }
4517
4518             } else { # cdrom
4519
4520                 if ($drive->{file} eq 'none') {
4521                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4522                 } else {
4523                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4524                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4525                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4526                 }
4527
4528                 return 1;
4529             }
4530         }
4531     }
4532
4533     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4534     # hotplug new disks
4535     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4536     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4537 }
4538
4539 sub vm_start {
4540     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4541         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4542
4543     PVE::QemuConfig->lock_config($vmid, sub {
4544         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4545
4546         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4547
4548         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4549
4550         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4551
4552         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4553             vmconfig_apply_pending($vmid, $conf, $storecfg);
4554             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4555         }
4556
4557         my $defaults = load_defaults();
4558
4559         # set environment variable useful inside network script
4560         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4561
4562         my $local_volumes = {};
4563
4564         if ($targetstorage) {
4565             foreach_drive($conf, sub {
4566                 my ($ds, $drive) = @_;
4567
4568                 return if drive_is_cdrom($drive);
4569
4570                 my $volid = $drive->{file};
4571
4572                 return if !$volid;
4573
4574                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4575
4576                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4577                 return if $scfg->{shared};
4578                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4579             });
4580
4581             my $format = undef;
4582
4583             foreach my $opt (sort keys %$local_volumes) {
4584
4585                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4586                 my $drive = parse_drive($opt, $conf->{$opt});
4587
4588                 #if remote storage is specified, use default format
4589                 if ($targetstorage && $targetstorage ne "1") {
4590                     $storeid = $targetstorage;
4591                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4592                     $format = $defFormat;
4593                 } else {
4594                     #else we use same format than original
4595                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4596                     $format = qemu_img_format($scfg, $volid);
4597                 }
4598
4599                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4600                 my $newdrive = $drive;
4601                 $newdrive->{format} = $format;
4602                 $newdrive->{file} = $newvolid;
4603                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4604                 $local_volumes->{$opt} = $drivestr;
4605                 #pass drive to conf for command line
4606                 $conf->{$opt} = $drivestr;
4607             }
4608         }
4609
4610         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4611
4612         my $migrate_port = 0;
4613         my $migrate_uri;
4614         if ($statefile) {
4615             if ($statefile eq 'tcp') {
4616                 my $localip = "localhost";
4617                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4618                 my $nodename = PVE::INotify::nodename();
4619
4620                 if (!defined($migration_type)) {
4621                     if (defined($datacenterconf->{migration}->{type})) {
4622                         $migration_type = $datacenterconf->{migration}->{type};
4623                     } else {
4624                         $migration_type = 'secure';
4625                     }
4626                 }
4627
4628                 if ($migration_type eq 'insecure') {
4629                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4630                     if ($migrate_network_addr) {
4631                         $localip = $migrate_network_addr;
4632                     } else {
4633                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4634                     }
4635
4636                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4637                 }
4638
4639                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4640                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4641                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4642                 push @$cmd, '-incoming', $migrate_uri;
4643                 push @$cmd, '-S';
4644
4645             } elsif ($statefile eq 'unix') {
4646                 # should be default for secure migrations as a ssh TCP forward
4647                 # tunnel is not deterministic reliable ready and fails regurarly
4648                 # to set up in time, so use UNIX socket forwards
4649                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4650                 unlink $socket_addr;
4651
4652                 $migrate_uri = "unix:$socket_addr";
4653
4654                 push @$cmd, '-incoming', $migrate_uri;
4655                 push @$cmd, '-S';
4656
4657             } else {
4658                 push @$cmd, '-loadstate', $statefile;
4659             }
4660         } elsif ($paused) {
4661             push @$cmd, '-S';
4662         }
4663
4664         # host pci devices
4665         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4666           my $d = parse_hostpci($conf->{"hostpci$i"});
4667           next if !$d;
4668           my $pcidevices = $d->{pciid};
4669           foreach my $pcidevice (@$pcidevices) {
4670                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4671
4672                 my $info = pci_device_info("0000:$pciid");
4673                 die "IOMMU not present\n" if !check_iommu_support();
4674                 die "no pci device info for device '$pciid'\n" if !$info;
4675                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4676                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4677           }
4678         }
4679
4680         PVE::Storage::activate_volumes($storecfg, $vollist);
4681
4682         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4683             my $cmd = [];
4684             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4685             eval  { run_command($cmd); };
4686         }
4687
4688         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4689                                                   : $defaults->{cpuunits};
4690
4691         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4692         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4693
4694         my %properties = (
4695             Slice => 'qemu.slice',
4696             KillMode => 'none',
4697             CPUShares => $cpuunits
4698         );
4699
4700         if (my $cpulimit = $conf->{cpulimit}) {
4701             $properties{CPUQuota} = int($cpulimit * 100);
4702         }
4703         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4704
4705         if ($conf->{hugepages}) {
4706
4707             my $code = sub {
4708                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4709                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4710
4711                 PVE::QemuServer::Memory::hugepages_mount();
4712                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4713
4714                 eval  {
4715                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4716                     run_command($cmd, %run_params);
4717                 };
4718
4719                 if (my $err = $@) {
4720                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4721                     die $err;
4722                 }
4723
4724                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4725             };
4726             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4727
4728         } else {
4729             eval  {
4730                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4731                 run_command($cmd, %run_params);
4732             };
4733         }
4734
4735         if (my $err = $@) {
4736             # deactivate volumes if start fails
4737             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4738             die "start failed: $err";
4739         }
4740
4741         print "migration listens on $migrate_uri\n" if $migrate_uri;
4742
4743         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4744             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4745             warn $@ if $@;
4746         }
4747
4748         #start nbd server for storage migration
4749         if ($targetstorage) {
4750             my $nodename = PVE::INotify::nodename();
4751             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4752             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4753             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4754             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4755
4756             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4757
4758             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4759
4760             foreach my $opt (sort keys %$local_volumes) {
4761                 my $volid = $local_volumes->{$opt};
4762                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4763                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4764                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4765             }
4766         }
4767
4768         if ($migratedfrom) {
4769             eval {
4770                 set_migration_caps($vmid);
4771             };
4772             warn $@ if $@;
4773
4774             if ($spice_port) {
4775                 print "spice listens on port $spice_port\n";
4776                 if ($spice_ticket) {
4777                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4778                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4779                 }
4780             }
4781
4782         } else {
4783             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4784                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4785                     if $conf->{balloon};
4786             }
4787
4788             foreach my $opt (keys %$conf) {
4789                 next if $opt !~  m/^net\d+$/;
4790                 my $nicconf = parse_net($conf->{$opt});
4791                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4792             }
4793         }
4794
4795         vm_mon_cmd_nocheck($vmid, 'qom-set',
4796                     path => "machine/peripheral/balloon0",
4797                     property => "guest-stats-polling-interval",
4798                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4799
4800     });
4801 }
4802
4803 sub vm_mon_cmd {
4804     my ($vmid, $execute, %params) = @_;
4805
4806     my $cmd = { execute => $execute, arguments => \%params };
4807     vm_qmp_command($vmid, $cmd);
4808 }
4809
4810 sub vm_mon_cmd_nocheck {
4811     my ($vmid, $execute, %params) = @_;
4812
4813     my $cmd = { execute => $execute, arguments => \%params };
4814     vm_qmp_command($vmid, $cmd, 1);
4815 }
4816
4817 sub vm_qmp_command {
4818     my ($vmid, $cmd, $nocheck) = @_;
4819
4820     my $res;
4821
4822     my $timeout;
4823     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4824         $timeout = $cmd->{arguments}->{timeout};
4825         delete $cmd->{arguments}->{timeout};
4826     }
4827
4828     eval {
4829         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4830         my $sname = qmp_socket($vmid);
4831         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4832             my $qmpclient = PVE::QMPClient->new();
4833
4834             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4835         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4836             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4837                 if scalar(%{$cmd->{arguments}});
4838             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4839         } else {
4840             die "unable to open monitor socket\n";
4841         }
4842     };
4843     if (my $err = $@) {
4844         syslog("err", "VM $vmid qmp command failed - $err");
4845         die $err;
4846     }
4847
4848     return $res;
4849 }
4850
4851 sub vm_human_monitor_command {
4852     my ($vmid, $cmdline) = @_;
4853
4854     my $res;
4855
4856     my $cmd = {
4857         execute => 'human-monitor-command',
4858         arguments => { 'command-line' => $cmdline},
4859     };
4860
4861     return vm_qmp_command($vmid, $cmd);
4862 }
4863
4864 sub vm_commandline {
4865     my ($storecfg, $vmid) = @_;
4866
4867     my $conf = PVE::QemuConfig->load_config($vmid);
4868
4869     my $defaults = load_defaults();
4870
4871     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4872
4873     return PVE::Tools::cmd2string($cmd);
4874 }
4875
4876 sub vm_reset {
4877     my ($vmid, $skiplock) = @_;
4878
4879     PVE::QemuConfig->lock_config($vmid, sub {
4880
4881         my $conf = PVE::QemuConfig->load_config($vmid);
4882
4883         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4884
4885         vm_mon_cmd($vmid, "system_reset");
4886     });
4887 }
4888
4889 sub get_vm_volumes {
4890     my ($conf) = @_;
4891
4892     my $vollist = [];
4893     foreach_volid($conf, sub {
4894         my ($volid, $attr) = @_;
4895
4896         return if $volid =~ m|^/|;
4897
4898         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4899         return if !$sid;
4900
4901         push @$vollist, $volid;
4902     });
4903
4904     return $vollist;
4905 }
4906
4907 sub vm_stop_cleanup {
4908     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4909
4910     eval {
4911
4912         if (!$keepActive) {
4913             my $vollist = get_vm_volumes($conf);
4914             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4915         }
4916
4917         foreach my $ext (qw(mon qmp pid vnc qga)) {
4918             unlink "/var/run/qemu-server/${vmid}.$ext";
4919         }
4920
4921         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4922     };
4923     warn $@ if $@; # avoid errors - just warn
4924 }
4925
4926 # Note: use $nockeck to skip tests if VM configuration file exists.
4927 # We need that when migration VMs to other nodes (files already moved)
4928 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4929 sub vm_stop {
4930     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4931
4932     $force = 1 if !defined($force) && !$shutdown;
4933
4934     if ($migratedfrom){
4935         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4936         kill 15, $pid if $pid;
4937         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4938         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4939         return;
4940     }
4941
4942     PVE::QemuConfig->lock_config($vmid, sub {
4943
4944         my $pid = check_running($vmid, $nocheck);
4945         return if !$pid;
4946
4947         my $conf;
4948         if (!$nocheck) {
4949             $conf = PVE::QemuConfig->load_config($vmid);
4950             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4951             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4952                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4953                 $timeout = $opts->{down} if $opts->{down};
4954             }
4955         }
4956
4957         $timeout = 60 if !defined($timeout);
4958
4959         eval {
4960             if ($shutdown) {
4961                 if (defined($conf) && $conf->{agent}) {
4962                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4963                 } else {
4964                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4965                 }
4966             } else {
4967                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4968             }
4969         };
4970         my $err = $@;
4971
4972         if (!$err) {
4973             my $count = 0;
4974             while (($count < $timeout) && check_running($vmid, $nocheck)) {
4975                 $count++;
4976                 sleep 1;
4977             }
4978
4979             if ($count >= $timeout) {
4980                 if ($force) {
4981                     warn "VM still running - terminating now with SIGTERM\n";
4982                     kill 15, $pid;
4983                 } else {
4984                     die "VM quit/powerdown failed - got timeout\n";
4985                 }
4986             } else {
4987                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
4988                 return;
4989             }
4990         } else {
4991             if ($force) {
4992                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
4993                 kill 15, $pid;
4994             } else {
4995                 die "VM quit/powerdown failed\n";
4996             }
4997         }
4998
4999         # wait again
5000         $timeout = 10;
5001
5002         my $count = 0;
5003         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5004             $count++;
5005             sleep 1;
5006         }
5007
5008         if ($count >= $timeout) {
5009             warn "VM still running - terminating now with SIGKILL\n";
5010             kill 9, $pid;
5011             sleep 1;
5012         }
5013
5014         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5015    });
5016 }
5017
5018 sub vm_suspend {
5019     my ($vmid, $skiplock) = @_;
5020
5021     PVE::QemuConfig->lock_config($vmid, sub {
5022
5023         my $conf = PVE::QemuConfig->load_config($vmid);
5024
5025         PVE::QemuConfig->check_lock($conf)
5026             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5027
5028         vm_mon_cmd($vmid, "stop");
5029     });
5030 }
5031
5032 sub vm_resume {
5033     my ($vmid, $skiplock, $nocheck) = @_;
5034
5035     PVE::QemuConfig->lock_config($vmid, sub {
5036
5037         if (!$nocheck) {
5038
5039             my $conf = PVE::QemuConfig->load_config($vmid);
5040
5041             PVE::QemuConfig->check_lock($conf)
5042                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5043
5044             vm_mon_cmd($vmid, "cont");
5045
5046         } else {
5047             vm_mon_cmd_nocheck($vmid, "cont");
5048         }
5049     });
5050 }
5051
5052 sub vm_sendkey {
5053     my ($vmid, $skiplock, $key) = @_;
5054
5055     PVE::QemuConfig->lock_config($vmid, sub {
5056
5057         my $conf = PVE::QemuConfig->load_config($vmid);
5058
5059         # there is no qmp command, so we use the human monitor command
5060         vm_human_monitor_command($vmid, "sendkey $key");
5061     });
5062 }
5063
5064 sub vm_destroy {
5065     my ($storecfg, $vmid, $skiplock) = @_;
5066
5067     PVE::QemuConfig->lock_config($vmid, sub {
5068
5069         my $conf = PVE::QemuConfig->load_config($vmid);
5070
5071         if (!check_running($vmid)) {
5072             destroy_vm($storecfg, $vmid, undef, $skiplock);
5073         } else {
5074             die "VM $vmid is running - destroy failed\n";
5075         }
5076     });
5077 }
5078
5079 # pci helpers
5080
5081 sub file_write {
5082     my ($filename, $buf) = @_;
5083
5084     my $fh = IO::File->new($filename, "w");
5085     return undef if !$fh;
5086
5087     my $res = print $fh $buf;
5088
5089     $fh->close();
5090
5091     return $res;
5092 }
5093
5094 sub pci_device_info {
5095     my ($name) = @_;
5096
5097     my $res;
5098
5099     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5100     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5101
5102     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5103     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5104
5105     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5106     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5107
5108     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5109     return undef if !defined($product) || $product !~ s/^0x//;
5110
5111     $res = {
5112         name => $name,
5113         vendor => $vendor,
5114         product => $product,
5115         domain => $domain,
5116         bus => $bus,
5117         slot => $slot,
5118         func => $func,
5119         irq => $irq,
5120         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5121     };
5122
5123     return $res;
5124 }
5125
5126 sub pci_dev_reset {
5127     my ($dev) = @_;
5128
5129     my $name = $dev->{name};
5130
5131     my $fn = "$pcisysfs/devices/$name/reset";
5132
5133     return file_write($fn, "1");
5134 }
5135
5136 sub pci_dev_bind_to_vfio {
5137     my ($dev) = @_;
5138
5139     my $name = $dev->{name};
5140
5141     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5142
5143     if (!-d $vfio_basedir) {
5144         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5145     }
5146     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5147
5148     my $testdir = "$vfio_basedir/$name";
5149     return 1 if -d $testdir;
5150
5151     my $data = "$dev->{vendor} $dev->{product}";
5152     return undef if !file_write("$vfio_basedir/new_id", $data);
5153
5154     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5155     if (!file_write($fn, $name)) {
5156         return undef if -f $fn;
5157     }
5158
5159     $fn = "$vfio_basedir/bind";
5160     if (! -d $testdir) {
5161         return undef if !file_write($fn, $name);
5162     }
5163
5164     return -d $testdir;
5165 }
5166
5167 sub pci_dev_group_bind_to_vfio {
5168     my ($pciid) = @_;
5169
5170     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5171
5172     if (!-d $vfio_basedir) {
5173         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5174     }
5175     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5176
5177     # get IOMMU group devices
5178     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5179       my @devs = grep /^0000:/, readdir($D);
5180     closedir($D);
5181
5182     foreach my $pciid (@devs) {
5183         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5184
5185         # pci bridges, switches or root ports are not supported
5186         # they have a pci_bus subdirectory so skip them
5187         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5188
5189         my $info = pci_device_info($1);
5190         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5191     }
5192
5193     return 1;
5194 }
5195
5196 # vzdump restore implementaion
5197
5198 sub tar_archive_read_firstfile {
5199     my $archive = shift;
5200
5201     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5202
5203     # try to detect archive type first
5204     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5205         die "unable to open file '$archive'\n";
5206     my $firstfile = <$fh>;
5207     kill 15, $pid;
5208     close $fh;
5209
5210     die "ERROR: archive contaions no data\n" if !$firstfile;
5211     chomp $firstfile;
5212
5213     return $firstfile;
5214 }
5215
5216 sub tar_restore_cleanup {
5217     my ($storecfg, $statfile) = @_;
5218
5219     print STDERR "starting cleanup\n";
5220
5221     if (my $fd = IO::File->new($statfile, "r")) {
5222         while (defined(my $line = <$fd>)) {
5223             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5224                 my $volid = $2;
5225                 eval {
5226                     if ($volid =~ m|^/|) {
5227                         unlink $volid || die 'unlink failed\n';
5228                     } else {
5229                         PVE::Storage::vdisk_free($storecfg, $volid);
5230                     }
5231                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5232                 };
5233                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5234             } else {
5235                 print STDERR "unable to parse line in statfile - $line";
5236             }
5237         }
5238         $fd->close();
5239     }
5240 }
5241
5242 sub restore_archive {
5243     my ($archive, $vmid, $user, $opts) = @_;
5244
5245     my $format = $opts->{format};
5246     my $comp;
5247
5248     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5249         $format = 'tar' if !$format;
5250         $comp = 'gzip';
5251     } elsif ($archive =~ m/\.tar$/) {
5252         $format = 'tar' if !$format;
5253     } elsif ($archive =~ m/.tar.lzo$/) {
5254         $format = 'tar' if !$format;
5255         $comp = 'lzop';
5256     } elsif ($archive =~ m/\.vma$/) {
5257         $format = 'vma' if !$format;
5258     } elsif ($archive =~ m/\.vma\.gz$/) {
5259         $format = 'vma' if !$format;
5260         $comp = 'gzip';
5261     } elsif ($archive =~ m/\.vma\.lzo$/) {
5262         $format = 'vma' if !$format;
5263         $comp = 'lzop';
5264     } else {
5265         $format = 'vma' if !$format; # default
5266     }
5267
5268     # try to detect archive format
5269     if ($format eq 'tar') {
5270         return restore_tar_archive($archive, $vmid, $user, $opts);
5271     } else {
5272         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5273     }
5274 }
5275
5276 sub restore_update_config_line {
5277     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5278
5279     return if $line =~ m/^\#qmdump\#/;
5280     return if $line =~ m/^\#vzdump\#/;
5281     return if $line =~ m/^lock:/;
5282     return if $line =~ m/^unused\d+:/;
5283     return if $line =~ m/^parent:/;
5284     return if $line =~ m/^template:/; # restored VM is never a template
5285
5286     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5287     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5288         # try to convert old 1.X settings
5289         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5290         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5291             my ($model, $macaddr) = split(/\=/, $devconfig);
5292             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5293             my $net = {
5294                 model => $model,
5295                 bridge => "vmbr$ind",
5296                 macaddr => $macaddr,
5297             };
5298             my $netstr = print_net($net);
5299
5300             print $outfd "net$cookie->{netcount}: $netstr\n";
5301             $cookie->{netcount}++;
5302         }
5303     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5304         my ($id, $netstr) = ($1, $2);
5305         my $net = parse_net($netstr);
5306         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5307         $netstr = print_net($net);
5308         print $outfd "$id: $netstr\n";
5309     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5310         my $virtdev = $1;
5311         my $value = $3;
5312         my $di = parse_drive($virtdev, $value);
5313         if (defined($di->{backup}) && !$di->{backup}) {
5314             print $outfd "#$line";
5315         } elsif ($map->{$virtdev}) {
5316             delete $di->{format}; # format can change on restore
5317             $di->{file} = $map->{$virtdev};
5318             $value = print_drive($vmid, $di);
5319             print $outfd "$virtdev: $value\n";
5320         } else {
5321             print $outfd $line;
5322         }
5323     } else {
5324         print $outfd $line;
5325     }
5326 }
5327
5328 sub scan_volids {
5329     my ($cfg, $vmid) = @_;
5330
5331     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5332
5333     my $volid_hash = {};
5334     foreach my $storeid (keys %$info) {
5335         foreach my $item (@{$info->{$storeid}}) {
5336             next if !($item->{volid} && $item->{size});
5337             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5338             $volid_hash->{$item->{volid}} = $item;
5339         }
5340     }
5341
5342     return $volid_hash;
5343 }
5344
5345 sub is_volume_in_use {
5346     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5347
5348     my $path = PVE::Storage::path($storecfg, $volid);
5349
5350     my $scan_config = sub {
5351         my ($cref, $snapname) = @_;
5352
5353         foreach my $key (keys %$cref) {
5354             my $value = $cref->{$key};
5355             if (is_valid_drivename($key)) {
5356                 next if $skip_drive && $key eq $skip_drive;
5357                 my $drive = parse_drive($key, $value);
5358                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5359                 return 1 if $volid eq $drive->{file};
5360                 if ($drive->{file} =~ m!^/!) {
5361                     return 1 if $drive->{file} eq $path;
5362                 } else {
5363                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5364                     next if !$storeid;
5365                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5366                     next if !$scfg;
5367                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5368                 }
5369             }
5370         }
5371
5372         return 0;
5373     };
5374
5375     return 1 if &$scan_config($conf);
5376
5377     undef $skip_drive;
5378
5379     foreach my $snapname (keys %{$conf->{snapshots}}) {
5380         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5381     }
5382
5383     return 0;
5384 }
5385
5386 sub update_disksize {
5387     my ($vmid, $conf, $volid_hash) = @_;
5388
5389     my $changes;
5390
5391     # used and unused disks
5392     my $referenced = {};
5393
5394     # Note: it is allowed to define multiple storages with same path (alias), so
5395     # we need to check both 'volid' and real 'path' (two different volid can point
5396     # to the same path).
5397
5398     my $referencedpath = {};
5399
5400     # update size info
5401     foreach my $opt (keys %$conf) {
5402         if (is_valid_drivename($opt)) {
5403             my $drive = parse_drive($opt, $conf->{$opt});
5404             my $volid = $drive->{file};
5405             next if !$volid;
5406
5407             $referenced->{$volid} = 1;
5408             if ($volid_hash->{$volid} &&
5409                 (my $path = $volid_hash->{$volid}->{path})) {
5410                 $referencedpath->{$path} = 1;
5411             }
5412
5413             next if drive_is_cdrom($drive);
5414             next if !$volid_hash->{$volid};
5415
5416             $drive->{size} = $volid_hash->{$volid}->{size};
5417             my $new = print_drive($vmid, $drive);
5418             if ($new ne $conf->{$opt}) {
5419                 $changes = 1;
5420                 $conf->{$opt} = $new;
5421             }
5422         }
5423     }
5424
5425     # remove 'unusedX' entry if volume is used
5426     foreach my $opt (keys %$conf) {
5427         next if $opt !~ m/^unused\d+$/;
5428         my $volid = $conf->{$opt};
5429         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5430         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5431             $changes = 1;
5432             delete $conf->{$opt};
5433         }
5434
5435         $referenced->{$volid} = 1;
5436         $referencedpath->{$path} = 1 if $path;
5437     }
5438
5439     foreach my $volid (sort keys %$volid_hash) {
5440         next if $volid =~ m/vm-$vmid-state-/;
5441         next if $referenced->{$volid};
5442         my $path = $volid_hash->{$volid}->{path};
5443         next if !$path; # just to be sure
5444         next if $referencedpath->{$path};
5445         $changes = 1;
5446         PVE::QemuConfig->add_unused_volume($conf, $volid);
5447         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5448     }
5449
5450     return $changes;
5451 }
5452
5453 sub rescan {
5454     my ($vmid, $nolock) = @_;
5455
5456     my $cfg = PVE::Storage::config();
5457
5458     my $volid_hash = scan_volids($cfg, $vmid);
5459
5460     my $updatefn =  sub {
5461         my ($vmid) = @_;
5462
5463         my $conf = PVE::QemuConfig->load_config($vmid);
5464
5465         PVE::QemuConfig->check_lock($conf);
5466
5467         my $vm_volids = {};
5468         foreach my $volid (keys %$volid_hash) {
5469             my $info = $volid_hash->{$volid};
5470             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5471         }
5472
5473         my $changes = update_disksize($vmid, $conf, $vm_volids);
5474
5475         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5476     };
5477
5478     if (defined($vmid)) {
5479         if ($nolock) {
5480             &$updatefn($vmid);
5481         } else {
5482             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5483         }
5484     } else {
5485         my $vmlist = config_list();
5486         foreach my $vmid (keys %$vmlist) {
5487             if ($nolock) {
5488                 &$updatefn($vmid);
5489             } else {
5490                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5491             }
5492         }
5493     }
5494 }
5495
5496 sub restore_vma_archive {
5497     my ($archive, $vmid, $user, $opts, $comp) = @_;
5498
5499     my $input = $archive eq '-' ? "<&STDIN" : undef;
5500     my $readfrom = $archive;
5501
5502     my $uncomp = '';
5503     if ($comp) {
5504         $readfrom = '-';
5505         my $qarchive = PVE::Tools::shellquote($archive);
5506         if ($comp eq 'gzip') {
5507             $uncomp = "zcat $qarchive|";
5508         } elsif ($comp eq 'lzop') {
5509             $uncomp = "lzop -d -c $qarchive|";
5510         } else {
5511             die "unknown compression method '$comp'\n";
5512         }
5513
5514     }
5515
5516     my $tmpdir = "/var/tmp/vzdumptmp$$";
5517     rmtree $tmpdir;
5518
5519     # disable interrupts (always do cleanups)
5520     local $SIG{INT} =
5521         local $SIG{TERM} =
5522         local $SIG{QUIT} =
5523         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5524
5525     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5526     POSIX::mkfifo($mapfifo, 0600);
5527     my $fifofh;
5528
5529     my $openfifo = sub {
5530         open($fifofh, '>', $mapfifo) || die $!;
5531     };
5532
5533     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5534
5535     my $oldtimeout;
5536     my $timeout = 5;
5537
5538     my $devinfo = {};
5539
5540     my $rpcenv = PVE::RPCEnvironment::get();
5541
5542     my $conffile = PVE::QemuConfig->config_file($vmid);
5543     my $tmpfn = "$conffile.$$.tmp";
5544
5545     # Note: $oldconf is undef if VM does not exists
5546     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5547     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5548
5549     my $print_devmap = sub {
5550         my $virtdev_hash = {};
5551
5552         my $cfgfn = "$tmpdir/qemu-server.conf";
5553
5554         # we can read the config - that is already extracted
5555         my $fh = IO::File->new($cfgfn, "r") ||
5556             "unable to read qemu-server.conf - $!\n";
5557
5558         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5559         if (-f $fwcfgfn) {
5560             my $pve_firewall_dir = '/etc/pve/firewall';
5561             mkdir $pve_firewall_dir; # make sure the dir exists
5562             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5563         }
5564
5565         while (defined(my $line = <$fh>)) {
5566             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5567                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5568                 die "archive does not contain data for drive '$virtdev'\n"
5569                     if !$devinfo->{$devname};
5570                 if (defined($opts->{storage})) {
5571                     $storeid = $opts->{storage} || 'local';
5572                 } elsif (!$storeid) {
5573                     $storeid = 'local';
5574                 }
5575                 $format = 'raw' if !$format;
5576                 $devinfo->{$devname}->{devname} = $devname;
5577                 $devinfo->{$devname}->{virtdev} = $virtdev;
5578                 $devinfo->{$devname}->{format} = $format;
5579                 $devinfo->{$devname}->{storeid} = $storeid;
5580
5581                 # check permission on storage
5582                 my $pool = $opts->{pool}; # todo: do we need that?
5583                 if ($user ne 'root@pam') {
5584                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5585                 }
5586
5587                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5588             }
5589         }
5590
5591         foreach my $devname (keys %$devinfo) {
5592             die "found no device mapping information for device '$devname'\n"
5593                 if !$devinfo->{$devname}->{virtdev};
5594         }
5595
5596         my $cfg = PVE::Storage::config();
5597
5598         # create empty/temp config
5599         if ($oldconf) {
5600             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5601             foreach_drive($oldconf, sub {
5602                 my ($ds, $drive) = @_;
5603
5604                 return if drive_is_cdrom($drive);
5605
5606                 my $volid = $drive->{file};
5607
5608                 return if !$volid || $volid =~ m|^/|;
5609
5610                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5611                 return if !$path || !$owner || ($owner != $vmid);
5612
5613                 # Note: only delete disk we want to restore
5614                 # other volumes will become unused
5615                 if ($virtdev_hash->{$ds}) {
5616                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5617                     if (my $err = $@) {
5618                         warn $err;
5619                     }
5620                 }
5621             });
5622
5623             # delete vmstate files
5624             # since after the restore we have no snapshots anymore
5625             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5626                 my $snap = $oldconf->{snapshots}->{$snapname};
5627                 if ($snap->{vmstate}) {
5628                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5629                     if (my $err = $@) {
5630                         warn $err;
5631                     }
5632                 }
5633             }
5634         }
5635
5636         my $map = {};
5637         foreach my $virtdev (sort keys %$virtdev_hash) {
5638             my $d = $virtdev_hash->{$virtdev};
5639             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5640             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5641
5642             # test if requested format is supported
5643             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5644             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5645             $d->{format} = $defFormat if !$supported;
5646
5647             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5648                                                   $d->{format}, undef, $alloc_size);
5649             print STDERR "new volume ID is '$volid'\n";
5650             $d->{volid} = $volid;
5651             my $path = PVE::Storage::path($cfg, $volid);
5652
5653             PVE::Storage::activate_volumes($cfg,[$volid]);
5654
5655             my $write_zeros = 1;
5656             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5657                 $write_zeros = 0;
5658             }
5659
5660             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5661
5662             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5663             $map->{$virtdev} = $volid;
5664         }
5665
5666         $fh->seek(0, 0) || die "seek failed - $!\n";
5667
5668         my $outfd = new IO::File ($tmpfn, "w") ||
5669             die "unable to write config for VM $vmid\n";
5670
5671         my $cookie = { netcount => 0 };
5672         while (defined(my $line = <$fh>)) {
5673             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5674         }
5675
5676         $fh->close();
5677         $outfd->close();
5678     };
5679
5680     eval {
5681         # enable interrupts
5682         local $SIG{INT} =
5683             local $SIG{TERM} =
5684             local $SIG{QUIT} =
5685             local $SIG{HUP} =
5686             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5687         local $SIG{ALRM} = sub { die "got timeout\n"; };
5688
5689         $oldtimeout = alarm($timeout);
5690
5691         my $parser = sub {
5692             my $line = shift;
5693
5694             print "$line\n";
5695
5696             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5697                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5698                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5699             } elsif ($line =~ m/^CTIME: /) {
5700                 # we correctly received the vma config, so we can disable
5701                 # the timeout now for disk allocation (set to 10 minutes, so
5702                 # that we always timeout if something goes wrong)
5703                 alarm(600);
5704                 &$print_devmap();
5705                 print $fifofh "done\n";
5706                 my $tmp = $oldtimeout || 0;
5707                 $oldtimeout = undef;
5708                 alarm($tmp);
5709                 close($fifofh);
5710             }
5711         };
5712
5713         print "restore vma archive: $cmd\n";
5714         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5715     };
5716     my $err = $@;
5717
5718     alarm($oldtimeout) if $oldtimeout;
5719
5720     my $vollist = [];
5721     foreach my $devname (keys %$devinfo) {
5722         my $volid = $devinfo->{$devname}->{volid};
5723         push @$vollist, $volid if $volid;
5724     }
5725
5726     my $cfg = PVE::Storage::config();
5727     PVE::Storage::deactivate_volumes($cfg, $vollist);
5728
5729     unlink $mapfifo;
5730
5731     if ($err) {
5732         rmtree $tmpdir;
5733         unlink $tmpfn;
5734
5735         foreach my $devname (keys %$devinfo) {
5736             my $volid = $devinfo->{$devname}->{volid};
5737             next if !$volid;
5738             eval {
5739                 if ($volid =~ m|^/|) {
5740                     unlink $volid || die 'unlink failed\n';
5741                 } else {
5742                     PVE::Storage::vdisk_free($cfg, $volid);
5743                 }
5744                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5745             };
5746             print STDERR "unable to cleanup '$volid' - $@" if $@;
5747         }
5748         die $err;
5749     }
5750
5751     rmtree $tmpdir;
5752
5753     rename($tmpfn, $conffile) ||
5754         die "unable to commit configuration file '$conffile'\n";
5755
5756     PVE::Cluster::cfs_update(); # make sure we read new file
5757
5758     eval { rescan($vmid, 1); };
5759     warn $@ if $@;
5760 }
5761
5762 sub restore_tar_archive {
5763     my ($archive, $vmid, $user, $opts) = @_;
5764
5765     if ($archive ne '-') {
5766         my $firstfile = tar_archive_read_firstfile($archive);
5767         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5768             if $firstfile ne 'qemu-server.conf';
5769     }
5770
5771     my $storecfg = PVE::Storage::config();
5772
5773     # destroy existing data - keep empty config
5774     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5775     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5776
5777     my $tocmd = "/usr/lib/qemu-server/qmextract";
5778
5779     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5780     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5781     $tocmd .= ' --prealloc' if $opts->{prealloc};
5782     $tocmd .= ' --info' if $opts->{info};
5783
5784     # tar option "xf" does not autodetect compression when read from STDIN,
5785     # so we pipe to zcat
5786     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5787         PVE::Tools::shellquote("--to-command=$tocmd");
5788
5789     my $tmpdir = "/var/tmp/vzdumptmp$$";
5790     mkpath $tmpdir;
5791
5792     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5793     local $ENV{VZDUMP_VMID} = $vmid;
5794     local $ENV{VZDUMP_USER} = $user;
5795
5796     my $conffile = PVE::QemuConfig->config_file($vmid);
5797     my $tmpfn = "$conffile.$$.tmp";
5798
5799     # disable interrupts (always do cleanups)
5800     local $SIG{INT} =
5801         local $SIG{TERM} =
5802         local $SIG{QUIT} =
5803         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5804
5805     eval {
5806         # enable interrupts
5807         local $SIG{INT} =
5808             local $SIG{TERM} =
5809             local $SIG{QUIT} =
5810             local $SIG{HUP} =
5811             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5812
5813         if ($archive eq '-') {
5814             print "extracting archive from STDIN\n";
5815             run_command($cmd, input => "<&STDIN");
5816         } else {
5817             print "extracting archive '$archive'\n";
5818             run_command($cmd);
5819         }
5820
5821         return if $opts->{info};
5822
5823         # read new mapping
5824         my $map = {};
5825         my $statfile = "$tmpdir/qmrestore.stat";
5826         if (my $fd = IO::File->new($statfile, "r")) {
5827             while (defined (my $line = <$fd>)) {
5828                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5829                     $map->{$1} = $2 if $1;
5830                 } else {
5831                     print STDERR "unable to parse line in statfile - $line\n";
5832                 }
5833             }
5834             $fd->close();
5835         }
5836
5837         my $confsrc = "$tmpdir/qemu-server.conf";
5838
5839         my $srcfd = new IO::File($confsrc, "r") ||
5840             die "unable to open file '$confsrc'\n";
5841
5842         my $outfd = new IO::File ($tmpfn, "w") ||
5843             die "unable to write config for VM $vmid\n";
5844
5845         my $cookie = { netcount => 0 };
5846         while (defined (my $line = <$srcfd>)) {
5847             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5848         }
5849
5850         $srcfd->close();
5851         $outfd->close();
5852     };
5853     my $err = $@;
5854
5855     if ($err) {
5856
5857         unlink $tmpfn;
5858
5859         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5860
5861         die $err;
5862     }
5863
5864     rmtree $tmpdir;
5865
5866     rename $tmpfn, $conffile ||
5867         die "unable to commit configuration file '$conffile'\n";
5868
5869     PVE::Cluster::cfs_update(); # make sure we read new file
5870
5871     eval { rescan($vmid, 1); };
5872     warn $@ if $@;
5873 };
5874
5875 sub foreach_storage_used_by_vm {
5876     my ($conf, $func) = @_;
5877
5878     my $sidhash = {};
5879
5880     foreach_drive($conf, sub {
5881         my ($ds, $drive) = @_;
5882         return if drive_is_cdrom($drive);
5883
5884         my $volid = $drive->{file};
5885
5886         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5887         $sidhash->{$sid} = $sid if $sid;
5888     });
5889
5890     foreach my $sid (sort keys %$sidhash) {
5891         &$func($sid);
5892     }
5893 }
5894
5895 sub do_snapshots_with_qemu {
5896     my ($storecfg, $volid) = @_;
5897
5898     my $storage_name = PVE::Storage::parse_volume_id($volid);
5899
5900     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5901         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5902         return 1;
5903     }
5904
5905     if ($volid =~ m/\.(qcow2|qed)$/){
5906         return 1;
5907     }
5908
5909     return undef;
5910 }
5911
5912 sub qga_check_running {
5913     my ($vmid) = @_;
5914
5915     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5916     if ($@) {
5917         warn "Qemu Guest Agent is not running - $@";
5918         return 0;
5919     }
5920     return 1;
5921 }
5922
5923 sub template_create {
5924     my ($vmid, $conf, $disk) = @_;
5925
5926     my $storecfg = PVE::Storage::config();
5927
5928     foreach_drive($conf, sub {
5929         my ($ds, $drive) = @_;
5930
5931         return if drive_is_cdrom($drive);
5932         return if $disk && $ds ne $disk;
5933
5934         my $volid = $drive->{file};
5935         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5936
5937         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5938         $drive->{file} = $voliddst;
5939         $conf->{$ds} = print_drive($vmid, $drive);
5940         PVE::QemuConfig->write_config($vmid, $conf);
5941     });
5942 }
5943
5944 sub qemu_img_convert {
5945     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5946
5947     my $storecfg = PVE::Storage::config();
5948     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5949     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5950
5951     if ($src_storeid && $dst_storeid) {
5952
5953         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5954
5955         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5956         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5957
5958         my $src_format = qemu_img_format($src_scfg, $src_volname);
5959         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5960
5961         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5962         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5963
5964         my $cmd = [];
5965         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5966         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5967         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5968         if ($is_zero_initialized) {
5969             push @$cmd, "zeroinit:$dst_path";
5970         } else {
5971             push @$cmd, $dst_path;
5972         }
5973
5974         my $parser = sub {
5975             my $line = shift;
5976             if($line =~ m/\((\S+)\/100\%\)/){
5977                 my $percent = $1;
5978                 my $transferred = int($size * $percent / 100);
5979                 my $remaining = $size - $transferred;
5980
5981                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
5982             }
5983
5984         };
5985
5986         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
5987         my $err = $@;
5988         die "copy failed: $err" if $err;
5989     }
5990 }
5991
5992 sub qemu_img_format {
5993     my ($scfg, $volname) = @_;
5994
5995     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
5996         return $1;
5997     } else {
5998         return "raw";
5999     }
6000 }
6001
6002 sub qemu_drive_mirror {
6003     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6004
6005     $jobs = {} if !$jobs;
6006
6007     my $qemu_target;
6008     my $format;
6009     $jobs->{"drive-$drive"} = {};
6010
6011     if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
6012         my $server = $1;
6013         my $port = $2;
6014         my $exportname = $3;
6015
6016         $format = "nbd";
6017         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
6018         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
6019         my $cmd = ['socat', '-T30', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=5"];
6020
6021         my $pid = fork();
6022         if (!defined($pid)) {
6023             die "forking socat tunnel failed\n";
6024         } elsif ($pid == 0) {
6025             exec(@$cmd);
6026             warn "exec failed: $!\n";
6027             POSIX::_exit(-1);
6028         }
6029         $jobs->{"drive-$drive"}->{pid} = $pid;
6030
6031         my $timeout = 0;
6032         while (!-S $unixsocket) {
6033             die "nbd connection helper timed out\n"
6034                 if $timeout++ > 5;
6035             sleep 1;
6036         }
6037     } else {
6038         my $storecfg = PVE::Storage::config();
6039         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6040
6041         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6042
6043         $format = qemu_img_format($dst_scfg, $dst_volname);
6044
6045         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6046
6047         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6048     }
6049
6050     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6051     $opts->{format} = $format if $format;
6052
6053     print "drive mirror is starting for drive-$drive\n";
6054
6055     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6056
6057     if (my $err = $@) {
6058         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6059         die "mirroring error: $err";
6060     }
6061
6062     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6063 }
6064
6065 sub qemu_drive_mirror_monitor {
6066     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6067
6068     eval {
6069         my $err_complete = 0;
6070
6071         while (1) {
6072             die "storage migration timed out\n" if $err_complete > 300;
6073
6074             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6075
6076             my $running_mirror_jobs = {};
6077             foreach my $stat (@$stats) {
6078                 next if $stat->{type} ne 'mirror';
6079                 $running_mirror_jobs->{$stat->{device}} = $stat;
6080             }
6081
6082             my $readycounter = 0;
6083
6084             foreach my $job (keys %$jobs) {
6085
6086                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6087                     print "$job : finished\n";
6088                     delete $jobs->{$job};
6089                     next;
6090                 }
6091
6092                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6093
6094                 my $busy = $running_mirror_jobs->{$job}->{busy};
6095                 my $ready = $running_mirror_jobs->{$job}->{ready};
6096                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6097                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6098                     my $remaining = $total - $transferred;
6099                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6100
6101                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6102                 }
6103
6104                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6105             }
6106
6107             last if scalar(keys %$jobs) == 0;
6108
6109             if ($readycounter == scalar(keys %$jobs)) {
6110                 print "all mirroring jobs are ready \n";
6111                 last if $skipcomplete; #do the complete later
6112
6113                 if ($vmiddst && $vmiddst != $vmid) {
6114                     my $agent_running = $qga && qga_check_running($vmid);
6115                     if ($agent_running) {
6116                         print "freeze filesystem\n";
6117                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6118                     } else {
6119                         print "suspend vm\n";
6120                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6121                     }
6122
6123                     # if we clone a disk for a new target vm, we don't switch the disk
6124                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6125
6126                     if ($agent_running) {
6127                         print "unfreeze filesystem\n";
6128                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6129                     } else {
6130                         print "resume vm\n";
6131                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6132                     }
6133
6134                     last;
6135                 } else {
6136
6137                     foreach my $job (keys %$jobs) {
6138                         # try to switch the disk if source and destination are on the same guest
6139                         print "$job: Completing block job...\n";
6140
6141                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6142                         if ($@ =~ m/cannot be completed/) {
6143                             print "$job: Block job cannot be completed, try again.\n";
6144                             $err_complete++;
6145                         }else {
6146                             print "$job: Completed successfully.\n";
6147                             $jobs->{$job}->{complete} = 1;
6148                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6149                         }
6150                     }
6151                 }
6152             }
6153             sleep 1;
6154         }
6155     };
6156     my $err = $@;
6157
6158     if ($err) {
6159         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6160         die "mirroring error: $err";
6161     }
6162
6163 }
6164
6165 sub qemu_blockjobs_cancel {
6166     my ($vmid, $jobs) = @_;
6167
6168     foreach my $job (keys %$jobs) {
6169         print "$job: Cancelling block job\n";
6170         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6171         $jobs->{$job}->{cancel} = 1;
6172     }
6173
6174     while (1) {
6175         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6176
6177         my $running_jobs = {};
6178         foreach my $stat (@$stats) {
6179             $running_jobs->{$stat->{device}} = $stat;
6180         }
6181
6182         foreach my $job (keys %$jobs) {
6183
6184             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6185                 print "$job: Done.\n";
6186                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6187                 delete $jobs->{$job};
6188             }
6189         }
6190
6191         last if scalar(keys %$jobs) == 0;
6192
6193         sleep 1;
6194     }
6195 }
6196
6197 sub qemu_blockjobs_finish_tunnel {
6198    my ($vmid, $job, $cpid) = @_;
6199
6200    return if !$cpid;
6201
6202    for (my $i = 1; $i < 20; $i++) {
6203         my $waitpid = waitpid($cpid, WNOHANG);
6204         last if (defined($waitpid) && ($waitpid == $cpid));
6205
6206         if ($i == 10) {
6207             kill(15, $cpid);
6208         } elsif ($i >= 15) {
6209             kill(9, $cpid);
6210         }
6211         sleep (1);
6212     }
6213     unlink "/run/qemu-server/$vmid.mirror-$job";
6214 }
6215
6216 sub clone_disk {
6217     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6218         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6219
6220     my $newvolid;
6221
6222     if (!$full) {
6223         print "create linked clone of drive $drivename ($drive->{file})\n";
6224         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6225         push @$newvollist, $newvolid;
6226     } else {
6227
6228         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6229         $storeid = $storage if $storage;
6230
6231         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6232         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6233
6234         print "create full clone of drive $drivename ($drive->{file})\n";
6235         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, undef, ($size/1024));
6236         push @$newvollist, $newvolid;
6237
6238         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6239
6240         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6241         if (!$running || $snapname) {
6242             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6243         } else {
6244
6245             my $kvmver = get_running_qemu_version ($vmid);
6246             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6247                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6248                     if $drive->{iothread};
6249             }
6250
6251             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6252         }
6253     }
6254
6255     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6256
6257     my $disk = $drive;
6258     $disk->{format} = undef;
6259     $disk->{file} = $newvolid;
6260     $disk->{size} = $size;
6261
6262     return $disk;
6263 }
6264
6265 # this only works if VM is running
6266 sub get_current_qemu_machine {
6267     my ($vmid) = @_;
6268
6269     my $cmd = { execute => 'query-machines', arguments => {} };
6270     my $res = vm_qmp_command($vmid, $cmd);
6271
6272     my ($current, $default);
6273     foreach my $e (@$res) {
6274         $default = $e->{name} if $e->{'is-default'};
6275         $current = $e->{name} if $e->{'is-current'};
6276     }
6277
6278     # fallback to the default machine if current is not supported by qemu
6279     return $current || $default || 'pc';
6280 }
6281
6282 sub get_running_qemu_version {
6283     my ($vmid) = @_;
6284     my $cmd = { execute => 'query-version', arguments => {} };
6285     my $res = vm_qmp_command($vmid, $cmd);
6286     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6287 }
6288
6289 sub qemu_machine_feature_enabled {
6290     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6291
6292     my $current_major;
6293     my $current_minor;
6294
6295     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6296
6297         $current_major = $3;
6298         $current_minor = $4;
6299
6300     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6301
6302         $current_major = $1;
6303         $current_minor = $2;
6304     }
6305
6306     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6307
6308
6309 }
6310
6311 sub qemu_machine_pxe {
6312     my ($vmid, $conf, $machine) = @_;
6313
6314     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6315
6316     foreach my $opt (keys %$conf) {
6317         next if $opt !~ m/^net(\d+)$/;
6318         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6319         next if !$net;
6320         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6321         return $machine.".pxe" if $romfile =~ m/pxe/;
6322         last;
6323     }
6324
6325     return $machine;
6326 }
6327
6328 sub qemu_use_old_bios_files {
6329     my ($machine_type) = @_;
6330
6331     return if !$machine_type;
6332
6333     my $use_old_bios_files = undef;
6334
6335     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6336         $machine_type = $1;
6337         $use_old_bios_files = 1;
6338     } else {
6339         my $kvmver = kvm_user_version();
6340         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6341         # load new efi bios files on migration. So this hack is required to allow
6342         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6343         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6344         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6345     }
6346
6347     return ($use_old_bios_files, $machine_type);
6348 }
6349
6350 sub create_efidisk {
6351     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6352
6353     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6354
6355     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6356     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6357     PVE::Storage::activate_volumes($storecfg, [$volid]);
6358
6359     my $path = PVE::Storage::path($storecfg, $volid);
6360     eval {
6361         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6362     };
6363     die "Copying EFI vars image failed: $@" if $@;
6364
6365     return ($volid, $vars_size);
6366 }
6367
6368 sub lspci {
6369
6370     my $devices = {};
6371
6372     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6373             my (undef, $id, $function) = @_;
6374             my $res = { id => $id, function => $function};
6375             push @{$devices->{$id}}, $res;
6376     });
6377
6378     # Entries should be sorted by functions.
6379     foreach my $id (keys %$devices) {
6380         my $dev = $devices->{$id};
6381         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6382     }
6383
6384     return $devices;
6385 }
6386
6387 sub vm_iothreads_list {
6388     my ($vmid) = @_;
6389
6390     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6391
6392     my $iothreads = {};
6393     foreach my $iothread (@$res) {
6394         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6395     }
6396
6397     return $iothreads;
6398 }
6399
6400 sub scsihw_infos {
6401     my ($conf, $drive) = @_;
6402
6403     my $maxdev = 0;
6404
6405     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6406         $maxdev = 7;
6407     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6408         $maxdev = 1;
6409     } else {
6410         $maxdev = 256;
6411     }
6412
6413     my $controller = int($drive->{index} / $maxdev);
6414     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6415
6416     return ($maxdev, $controller, $controller_prefix);
6417 }
6418
6419 sub add_hyperv_enlightenments {
6420     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6421
6422     return if $winversion < 6;
6423     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6424
6425     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6426
6427     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6428         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6429         push @$cpuFlags , 'hv_vapic';
6430         push @$cpuFlags , 'hv_time';
6431     } else {
6432         push @$cpuFlags , 'hv_spinlocks=0xffff';
6433     }
6434
6435     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6436         push @$cpuFlags , 'hv_reset';
6437         push @$cpuFlags , 'hv_vpindex';
6438         push @$cpuFlags , 'hv_runtime';
6439     }
6440
6441     if ($winversion >= 7) {
6442         push @$cpuFlags , 'hv_relaxed';
6443     }
6444 }
6445
6446 sub windows_version {
6447     my ($ostype) = @_;
6448
6449     return 0 if !$ostype;
6450
6451     my $winversion = 0;
6452
6453     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6454         $winversion = 5;
6455     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6456         $winversion = 6;
6457     } elsif ($ostype =~ m/^win(\d+)$/) {
6458         $winversion = $1;
6459     }
6460
6461     return $winversion;
6462 }
6463
6464 sub resolve_dst_disk_format {
6465         my ($storecfg, $storeid, $src_volname, $format) = @_;
6466         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6467
6468         if (!$format) {
6469             # if no target format is specified, use the source disk format as hint
6470             if ($src_volname) {
6471                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6472                 $format = qemu_img_format($scfg, $src_volname);
6473             } else {
6474                 return $defFormat;
6475             }
6476         }
6477
6478         # test if requested format is supported - else use default
6479         my $supported = grep { $_ eq $format } @$validFormats;
6480         $format = $defFormat if !$supported;
6481         return $format;
6482 }
6483
6484 sub resolve_first_disk {
6485     my $conf = shift;
6486     my @disks = PVE::QemuServer::valid_drive_names();
6487     my $firstdisk;
6488     foreach my $ds (reverse @disks) {
6489         next if !$conf->{$ds};
6490         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6491         next if PVE::QemuServer::drive_is_cdrom($disk);
6492         $firstdisk = $ds;
6493     }
6494     return $firstdisk;
6495 }
6496
6497 sub generate_smbios1_uuid {
6498     my ($uuid, $uuid_str);
6499     UUID::generate($uuid);
6500     UUID::unparse($uuid, $uuid_str);
6501     return "uuid=$uuid_str";
6502 }
6503
6504 # bash completion helper
6505
6506 sub complete_backup_archives {
6507     my ($cmdname, $pname, $cvalue) = @_;
6508
6509     my $cfg = PVE::Storage::config();
6510
6511     my $storeid;
6512
6513     if ($cvalue =~ m/^([^:]+):/) {
6514         $storeid = $1;
6515     }
6516
6517     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6518
6519     my $res = [];
6520     foreach my $id (keys %$data) {
6521         foreach my $item (@{$data->{$id}}) {
6522             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6523             push @$res, $item->{volid} if defined($item->{volid});
6524         }
6525     }
6526
6527     return $res;
6528 }
6529
6530 my $complete_vmid_full = sub {
6531     my ($running) = @_;
6532
6533     my $idlist = vmstatus();
6534
6535     my $res = [];
6536
6537     foreach my $id (keys %$idlist) {
6538         my $d = $idlist->{$id};
6539         if (defined($running)) {
6540             next if $d->{template};
6541             next if $running && $d->{status} ne 'running';
6542             next if !$running && $d->{status} eq 'running';
6543         }
6544         push @$res, $id;
6545
6546     }
6547     return $res;
6548 };
6549
6550 sub complete_vmid {
6551     return &$complete_vmid_full();
6552 }
6553
6554 sub complete_vmid_stopped {
6555     return &$complete_vmid_full(0);
6556 }
6557
6558 sub complete_vmid_running {
6559     return &$complete_vmid_full(1);
6560 }
6561
6562 sub complete_storage {
6563
6564     my $cfg = PVE::Storage::config();
6565     my $ids = $cfg->{ids};
6566
6567     my $res = [];
6568     foreach my $sid (keys %$ids) {
6569         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6570         next if !$ids->{$sid}->{content}->{images};
6571         push @$res, $sid;
6572     }
6573
6574     return $res;
6575 }
6576
6577 sub nbd_stop {
6578     my ($vmid) = @_;
6579
6580     vm_mon_cmd($vmid, 'nbd-server-stop');
6581 }
6582
6583 1;