PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42
  43 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  44
  45 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  46
  47 # Note about locking: we use flock on the config file protect
  48 # against concurent actions.
  49 # Aditionaly, we have a 'lock' setting in the config file. This
  50 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  51 # allowed when such lock is set. But you can ignore this kind of
  52 # lock with the --skiplock flag.
  53
  54 cfs_register_file('/qemu-server/',
  55                   \&parse_vm_config,
  56                   \&write_vm_config);
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  72     type => 'string',
  73     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  74     description => "The drive's backing file's data format.",
  75     optional => 1,
  76 });
  77
  78 #no warnings 'redefine';
  79
  80 sub cgroups_write {
  81    my ($controller, $vmid, $option, $value) = @_;
  82
  83    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  84    PVE::ProcFSTools::write_proc_entry($path, $value);
  85
  86 }
  87
  88 my $nodename = PVE::INotify::nodename();
  89
  90 mkdir "/etc/pve/nodes/$nodename";
  91 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  92 mkdir $confdir;
  93
  94 my $var_run_tmpdir = "/var/run/qemu-server";
  95 mkdir $var_run_tmpdir;
  96
  97 my $lock_dir = "/var/lock/qemu-server";
  98 mkdir $lock_dir;
  99
 100 my $pcisysfs = "/sys/bus/pci";
 101
 102 my $cpu_vendor_list = {
 103     # Intel CPUs
 104     486 => 'GenuineIntel',
 105     pentium => 'GenuineIntel',
 106     pentium2  => 'GenuineIntel',
 107     pentium3  => 'GenuineIntel',
 108     coreduo => 'GenuineIntel',
 109     core2duo => 'GenuineIntel',
 110     Conroe  => 'GenuineIntel',
 111     Penryn  => 'GenuineIntel',
 112     Nehalem  => 'GenuineIntel',
 113     'Nehalem-IBRS'  => 'GenuineIntel',
 114     Westmere => 'GenuineIntel',
 115     'Westmere-IBRS' => 'GenuineIntel',
 116     SandyBridge => 'GenuineIntel',
 117     'SandyBridge-IBRS' => 'GenuineIntel',
 118     IvyBridge => 'GenuineIntel',
 119     'IvyBridge-IBRS' => 'GenuineIntel',
 120     Haswell => 'GenuineIntel',
 121     'Haswell-IBRS' => 'GenuineIntel',
 122     'Haswell-noTSX' => 'GenuineIntel',
 123     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 124     Broadwell => 'GenuineIntel',
 125     'Broadwell-IBRS' => 'GenuineIntel',
 126     'Broadwell-noTSX' => 'GenuineIntel',
 127     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 128     'Skylake-Client' => 'GenuineIntel',
 129     'Skylake-Client-IBRS' => 'GenuineIntel',
 130     'Skylake-Server' => 'GenuineIntel',
 131     'Skylake-Server-IBRS' => 'GenuineIntel',
 132
 133     # AMD CPUs
 134     athlon => 'AuthenticAMD',
 135     phenom  => 'AuthenticAMD',
 136     Opteron_G1  => 'AuthenticAMD',
 137     Opteron_G2  => 'AuthenticAMD',
 138     Opteron_G3  => 'AuthenticAMD',
 139     Opteron_G4  => 'AuthenticAMD',
 140     Opteron_G5  => 'AuthenticAMD',
 141
 142     # generic types, use vendor from host node
 143     host => 'default',
 144     kvm32 => 'default',
 145     kvm64 => 'default',
 146     qemu32 => 'default',
 147     qemu64 => 'default',
 148     max => 'default',
 149 };
 150
 151 my $cpu_flag = qr/[+-](pcid|spec-ctrl)/;
 152
 153 my $cpu_fmt = {
 154     cputype => {
 155         description => "Emulated CPU type.",
 156         type => 'string',
 157         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 158         default => 'kvm64',
 159         default_key => 1,
 160     },
 161     hidden => {
 162         description => "Do not identify as a KVM virtual machine.",
 163         type => 'boolean',
 164         optional => 1,
 165         default => 0
 166     },
 167     flags => {
 168         description => "List of additional CPU flags separated by ';'."
 169                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 170                      . " Currently supported flags: 'pcid', 'spec-ctrl'.",
 171         format_description => '+FLAG[;-FLAG...]',
 172         type => 'string',
 173         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 174         optional => 1,
 175     },
 176 };
 177
 178 my $watchdog_fmt = {
 179     model => {
 180         default_key => 1,
 181         type => 'string',
 182         enum => [qw(i6300esb ib700)],
 183         description => "Watchdog type to emulate.",
 184         default => 'i6300esb',
 185         optional => 1,
 186     },
 187     action => {
 188         type => 'string',
 189         enum => [qw(reset shutdown poweroff pause debug none)],
 190         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 191         optional => 1,
 192     },
 193 };
 194 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 195
 196 my $confdesc = {
 197     onboot => {
 198         optional => 1,
 199         type => 'boolean',
 200         description => "Specifies whether a VM will be started during system bootup.",
 201         default => 0,
 202     },
 203     autostart => {
 204         optional => 1,
 205         type => 'boolean',
 206         description => "Automatic restart after crash (currently ignored).",
 207         default => 0,
 208     },
 209     hotplug => {
 210         optional => 1,
 211         type => 'string', format => 'pve-hotplug-features',
 212         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 213         default => 'network,disk,usb',
 214     },
 215     reboot => {
 216         optional => 1,
 217         type => 'boolean',
 218         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 219         default => 1,
 220     },
 221     lock => {
 222         optional => 1,
 223         type => 'string',
 224         description => "Lock/unlock the VM.",
 225         enum => [qw(migrate backup snapshot rollback)],
 226     },
 227     cpulimit => {
 228         optional => 1,
 229         type => 'number',
 230         description => "Limit of CPU usage.",
 231         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 232         minimum => 0,
 233         maximum => 128,
 234         default => 0,
 235     },
 236     cpuunits => {
 237         optional => 1,
 238         type => 'integer',
 239         description => "CPU weight for a VM.",
 240         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 241         minimum => 2,
 242         maximum => 262144,
 243         default => 1024,
 244     },
 245     memory => {
 246         optional => 1,
 247         type => 'integer',
 248         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 249         minimum => 16,
 250         default => 512,
 251     },
 252     balloon => {
 253         optional => 1,
 254         type => 'integer',
 255         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 256         minimum => 0,
 257     },
 258     shares => {
 259         optional => 1,
 260         type => 'integer',
 261         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 262         minimum => 0,
 263         maximum => 50000,
 264         default => 1000,
 265     },
 266     keyboard => {
 267         optional => 1,
 268         type => 'string',
 269         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.".
 270                        "It should not be necessary to set it.",
 271         enum => PVE::Tools::kvmkeymaplist(),
 272         default => undef,
 273     },
 274     name => {
 275         optional => 1,
 276         type => 'string', format => 'dns-name',
 277         description => "Set a name for the VM. Only used on the configuration web interface.",
 278     },
 279     scsihw => {
 280         optional => 1,
 281         type => 'string',
 282         description => "SCSI controller model",
 283         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 284         default => 'lsi',
 285     },
 286     description => {
 287         optional => 1,
 288         type => 'string',
 289         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 290     },
 291     ostype => {
 292         optional => 1,
 293         type => 'string',
 294         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 295         description => "Specify guest operating system.",
 296         verbose_description => <<EODESC,
 297 Specify guest operating system. This is used to enable special
 298 optimization/features for specific operating systems:
 299
 300 [horizontal]
 301 other;; unspecified OS
 302 wxp;; Microsoft Windows XP
 303 w2k;; Microsoft Windows 2000
 304 w2k3;; Microsoft Windows 2003
 305 w2k8;; Microsoft Windows 2008
 306 wvista;; Microsoft Windows Vista
 307 win7;; Microsoft Windows 7
 308 win8;; Microsoft Windows 8/2012/2012r2
 309 win10;; Microsoft Windows 10/2016
 310 l24;; Linux 2.4 Kernel
 311 l26;; Linux 2.6/3.X Kernel
 312 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 313 EODESC
 314     },
 315     boot => {
 316         optional => 1,
 317         type => 'string',
 318         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 319         pattern => '[acdn]{1,4}',
 320         default => 'cdn',
 321     },
 322     bootdisk => {
 323         optional => 1,
 324         type => 'string', format => 'pve-qm-bootdisk',
 325         description => "Enable booting from specified disk.",
 326         pattern => '(ide|sata|scsi|virtio)\d+',
 327     },
 328     smp => {
 329         optional => 1,
 330         type => 'integer',
 331         description => "The number of CPUs. Please use option -sockets instead.",
 332         minimum => 1,
 333         default => 1,
 334     },
 335     sockets => {
 336         optional => 1,
 337         type => 'integer',
 338         description => "The number of CPU sockets.",
 339         minimum => 1,
 340         default => 1,
 341     },
 342     cores => {
 343         optional => 1,
 344         type => 'integer',
 345         description => "The number of cores per socket.",
 346         minimum => 1,
 347         default => 1,
 348     },
 349     numa => {
 350         optional => 1,
 351         type => 'boolean',
 352         description => "Enable/disable NUMA.",
 353         default => 0,
 354     },
 355     hugepages => {
 356         optional => 1,
 357         type => 'string',
 358         description => "Enable/disable hugepages memory.",
 359         enum => [qw(any 2 1024)],
 360     },
 361     vcpus => {
 362         optional => 1,
 363         type => 'integer',
 364         description => "Number of hotplugged vcpus.",
 365         minimum => 1,
 366         default => 0,
 367     },
 368     acpi => {
 369         optional => 1,
 370         type => 'boolean',
 371         description => "Enable/disable ACPI.",
 372         default => 1,
 373     },
 374     agent => {
 375         optional => 1,
 376         type => 'boolean',
 377         description => "Enable/disable Qemu GuestAgent.",
 378         default => 0,
 379     },
 380     kvm => {
 381         optional => 1,
 382         type => 'boolean',
 383         description => "Enable/disable KVM hardware virtualization.",
 384         default => 1,
 385     },
 386     tdf => {
 387         optional => 1,
 388         type => 'boolean',
 389         description => "Enable/disable time drift fix.",
 390         default => 0,
 391     },
 392     localtime => {
 393         optional => 1,
 394         type => 'boolean',
 395         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 396     },
 397     freeze => {
 398         optional => 1,
 399         type => 'boolean',
 400         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 401     },
 402     vga => {
 403         optional => 1,
 404         type => 'string',
 405         description => "Select the VGA type.",
 406         verbose_description => "Select the VGA type. If you want to use high resolution" .
 407             " modes (>= 1280x1024x16) then you should use the options " .
 408             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 409             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 410             "display sever. For win* OS you can select how many independent " .
 411             "displays you want, Linux guests can add displays them self. " .
 412             "You can also run without any graphic card, using a serial device" .
 413             " as terminal.",
 414         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 415     },
 416     watchdog => {
 417         optional => 1,
 418         type => 'string', format => 'pve-qm-watchdog',
 419         description => "Create a virtual hardware watchdog device.",
 420         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 421             " (by a guest action), the watchdog must be periodically polled " .
 422             "by an agent inside the guest or else the watchdog will reset " .
 423             "the guest (or execute the respective action specified)",
 424     },
 425     startdate => {
 426         optional => 1,
 427         type => 'string',
 428         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 429         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 430         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 431         default => 'now',
 432     },
 433     startup =>  get_standard_option('pve-startup-order'),
 434     template => {
 435         optional => 1,
 436         type => 'boolean',
 437         description => "Enable/disable Template.",
 438         default => 0,
 439     },
 440     args => {
 441         optional => 1,
 442         type => 'string',
 443         description => "Arbitrary arguments passed to kvm.",
 444         verbose_description => <<EODESCR,
 445 Arbitrary arguments passed to kvm, for example:
 446
 447 args: -no-reboot -no-hpet
 448
 449 NOTE: this option is for experts only.
 450 EODESCR
 451     },
 452     tablet => {
 453         optional => 1,
 454         type => 'boolean',
 455         default => 1,
 456         description => "Enable/disable the USB tablet device.",
 457         verbose_description => "Enable/disable the USB tablet device. This device is " .
 458             "usually needed to allow absolute mouse positioning with VNC. " .
 459             "Else the mouse runs out of sync with normal VNC clients. " .
 460             "If you're running lots of console-only guests on one host, " .
 461             "you may consider disabling this to save some context switches. " .
 462             "This is turned off by default if you use spice (-vga=qxl).",
 463     },
 464     migrate_speed => {
 465         optional => 1,
 466         type => 'integer',
 467         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 468         minimum => 0,
 469         default => 0,
 470     },
 471     migrate_downtime => {
 472         optional => 1,
 473         type => 'number',
 474         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 475         minimum => 0,
 476         default => 0.1,
 477     },
 478     cdrom => {
 479         optional => 1,
 480         type => 'string', format => 'pve-qm-ide',
 481         typetext => '<volume>',
 482         description => "This is an alias for option -ide2",
 483     },
 484     cpu => {
 485         optional => 1,
 486         description => "Emulated CPU type.",
 487         type => 'string',
 488         format => $cpu_fmt,
 489     },
 490     parent => get_standard_option('pve-snapshot-name', {
 491         optional => 1,
 492         description => "Parent snapshot name. This is used internally, and should not be modified.",
 493     }),
 494     snaptime => {
 495         optional => 1,
 496         description => "Timestamp for snapshots.",
 497         type => 'integer',
 498         minimum => 0,
 499     },
 500     vmstate => {
 501         optional => 1,
 502         type => 'string', format => 'pve-volume-id',
 503         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 504     },
 505     vmstatestorage => get_standard_option('pve-storage-id', {
 506         description => "Default storage for VM state volumes/files.",
 507         optional => 1,
 508     }),
 509     machine => {
 510         description => "Specific the Qemu machine type.",
 511         type => 'string',
 512         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 513         maxLength => 40,
 514         optional => 1,
 515     },
 516     smbios1 => {
 517         description => "Specify SMBIOS type 1 fields.",
 518         type => 'string', format => 'pve-qm-smbios1',
 519         maxLength => 256,
 520         optional => 1,
 521     },
 522     protection => {
 523         optional => 1,
 524         type => 'boolean',
 525         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 526         default => 0,
 527     },
 528     bios => {
 529         optional => 1,
 530         type => 'string',
 531         enum => [ qw(seabios ovmf) ],
 532         description => "Select BIOS implementation.",
 533         default => 'seabios',
 534     },
 535 };
 536
 537 # what about other qemu settings ?
 538 #cpu => 'string',
 539 #machine => 'string',
 540 #fda => 'file',
 541 #fdb => 'file',
 542 #mtdblock => 'file',
 543 #sd => 'file',
 544 #pflash => 'file',
 545 #snapshot => 'bool',
 546 #bootp => 'file',
 547 ##tftp => 'dir',
 548 ##smb => 'dir',
 549 #kernel => 'file',
 550 #append => 'string',
 551 #initrd => 'file',
 552 ##soundhw => 'string',
 553
 554 while (my ($k, $v) = each %$confdesc) {
 555     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 556 }
 557
 558 my $MAX_IDE_DISKS = 4;
 559 my $MAX_SCSI_DISKS = 14;
 560 my $MAX_VIRTIO_DISKS = 16;
 561 my $MAX_SATA_DISKS = 6;
 562 my $MAX_USB_DEVICES = 5;
 563 my $MAX_NETS = 32;
 564 my $MAX_UNUSED_DISKS = 8;
 565 my $MAX_HOSTPCI_DEVICES = 4;
 566 my $MAX_SERIAL_PORTS = 4;
 567 my $MAX_PARALLEL_PORTS = 3;
 568 my $MAX_NUMA = 8;
 569
 570 my $numa_fmt = {
 571     cpus => {
 572         type => "string",
 573         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 574         description => "CPUs accessing this NUMA node.",
 575         format_description => "id[-id];...",
 576     },
 577     memory => {
 578         type => "number",
 579         description => "Amount of memory this NUMA node provides.",
 580         optional => 1,
 581     },
 582     hostnodes => {
 583         type => "string",
 584         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 585         description => "Host NUMA nodes to use.",
 586         format_description => "id[-id];...",
 587         optional => 1,
 588     },
 589     policy => {
 590         type => 'string',
 591         enum => [qw(preferred bind interleave)],
 592         description => "NUMA allocation policy.",
 593         optional => 1,
 594     },
 595 };
 596 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 597 my $numadesc = {
 598     optional => 1,
 599     type => 'string', format => $numa_fmt,
 600     description => "NUMA topology.",
 601 };
 602 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 603
 604 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 605     $confdesc->{"numa$i"} = $numadesc;
 606 }
 607
 608 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 609                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 610                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 611 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 612
 613 my $net_fmt_bridge_descr = <<__EOD__;
 614 Bridge to attach the network device to. The Proxmox VE standard bridge
 615 is called 'vmbr0'.
 616
 617 If you do not specify a bridge, we create a kvm user (NATed) network
 618 device, which provides DHCP and DNS services. The following addresses
 619 are used:
 620
 621  10.0.2.2   Gateway
 622  10.0.2.3   DNS Server
 623  10.0.2.4   SMB Server
 624
 625 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 626 __EOD__
 627
 628 my $net_fmt = {
 629     macaddr => {
 630         type => 'string',
 631         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 632         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 633         format_description => "XX:XX:XX:XX:XX:XX",
 634         optional => 1,
 635     },
 636     model => {
 637         type => 'string',
 638         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 639         enum => $nic_model_list,
 640         default_key => 1,
 641     },
 642     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 643     bridge => {
 644         type => 'string',
 645         description => $net_fmt_bridge_descr,
 646         format_description => 'bridge',
 647         optional => 1,
 648     },
 649     queues => {
 650         type => 'integer',
 651         minimum => 0, maximum => 16,
 652         description => 'Number of packet queues to be used on the device.',
 653         optional => 1,
 654     },
 655     rate => {
 656         type => 'number',
 657         minimum => 0,
 658         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 659         optional => 1,
 660     },
 661     tag => {
 662         type => 'integer',
 663         minimum => 1, maximum => 4094,
 664         description => 'VLAN tag to apply to packets on this interface.',
 665         optional => 1,
 666     },
 667     trunks => {
 668         type => 'string',
 669         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 670         description => 'VLAN trunks to pass through this interface.',
 671         format_description => 'vlanid[;vlanid...]',
 672         optional => 1,
 673     },
 674     firewall => {
 675         type => 'boolean',
 676         description => 'Whether this interface should be protected by the firewall.',
 677         optional => 1,
 678     },
 679     link_down => {
 680         type => 'boolean',
 681         description => 'Whether this interface should be disconnected (like pulling the plug).',
 682         optional => 1,
 683     },
 684 };
 685
 686 my $netdesc = {
 687     optional => 1,
 688     type => 'string', format => $net_fmt,
 689     description => "Specify network devices.",
 690 };
 691
 692 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 693
 694 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 695     $confdesc->{"net$i"} = $netdesc;
 696 }
 697
 698 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 699 sub verify_volume_id_or_qm_path {
 700     my ($volid, $noerr) = @_;
 701
 702     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 703         return $volid;
 704     }
 705
 706     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 707     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 708     if ($@) {
 709         return undef if $noerr;
 710         die $@;
 711     }
 712     return $volid;
 713 }
 714
 715 my $drivename_hash;
 716
 717 my %drivedesc_base = (
 718     volume => { alias => 'file' },
 719     file => {
 720         type => 'string',
 721         format => 'pve-volume-id-or-qm-path',
 722         default_key => 1,
 723         format_description => 'volume',
 724         description => "The drive's backing volume.",
 725     },
 726     media => {
 727         type => 'string',
 728         enum => [qw(cdrom disk)],
 729         description => "The drive's media type.",
 730         default => 'disk',
 731         optional => 1
 732     },
 733     cyls => {
 734         type => 'integer',
 735         description => "Force the drive's physical geometry to have a specific cylinder count.",
 736         optional => 1
 737     },
 738     heads => {
 739         type => 'integer',
 740         description => "Force the drive's physical geometry to have a specific head count.",
 741         optional => 1
 742     },
 743     secs => {
 744         type => 'integer',
 745         description => "Force the drive's physical geometry to have a specific sector count.",
 746         optional => 1
 747     },
 748     trans => {
 749         type => 'string',
 750         enum => [qw(none lba auto)],
 751         description => "Force disk geometry bios translation mode.",
 752         optional => 1,
 753     },
 754     snapshot => {
 755         type => 'boolean',
 756         description => "Whether the drive should be included when making snapshots.",
 757         optional => 1,
 758     },
 759     cache => {
 760         type => 'string',
 761         enum => [qw(none writethrough writeback unsafe directsync)],
 762         description => "The drive's cache mode",
 763         optional => 1,
 764     },
 765     format => get_standard_option('pve-qm-image-format'),
 766     size => {
 767         type => 'string',
 768         format => 'disk-size',
 769         format_description => 'DiskSize',
 770         description => "Disk size. This is purely informational and has no effect.",
 771         optional => 1,
 772     },
 773     backup => {
 774         type => 'boolean',
 775         description => "Whether the drive should be included when making backups.",
 776         optional => 1,
 777     },
 778     replicate => {
 779         type => 'boolean',
 780         description => 'Whether the drive should considered for replication jobs.',
 781         optional => 1,
 782         default => 1,
 783     },
 784     rerror => {
 785         type => 'string',
 786         enum => [qw(ignore report stop)],
 787         description => 'Read error action.',
 788         optional => 1,
 789     },
 790     werror => {
 791         type => 'string',
 792         enum => [qw(enospc ignore report stop)],
 793         description => 'Write error action.',
 794         optional => 1,
 795     },
 796     aio => {
 797         type => 'string',
 798         enum => [qw(native threads)],
 799         description => 'AIO type to use.',
 800         optional => 1,
 801     },
 802     discard => {
 803         type => 'string',
 804         enum => [qw(ignore on)],
 805         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 806         optional => 1,
 807     },
 808     detect_zeroes => {
 809         type => 'boolean',
 810         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 811         optional => 1,
 812     },
 813     serial => {
 814         type => 'string',
 815         format => 'urlencoded',
 816         format_description => 'serial',
 817         maxLength => 20*3, # *3 since it's %xx url enoded
 818         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 819         optional => 1,
 820     }
 821 );
 822
 823 my %iothread_fmt = ( iothread => {
 824         type => 'boolean',
 825         description => "Whether to use iothreads for this drive",
 826         optional => 1,
 827 });
 828
 829 my %model_fmt = (
 830     model => {
 831         type => 'string',
 832         format => 'urlencoded',
 833         format_description => 'model',
 834         maxLength => 40*3, # *3 since it's %xx url enoded
 835         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 836         optional => 1,
 837     },
 838 );
 839
 840 my %queues_fmt = (
 841     queues => {
 842         type => 'integer',
 843         description => "Number of queues.",
 844         minimum => 2,
 845         optional => 1
 846     }
 847 );
 848
 849 my %scsiblock_fmt = (
 850     scsiblock => {
 851         type => 'boolean',
 852         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 853         optional => 1,
 854         default => 0,
 855     },
 856 );
 857
 858 my $add_throttle_desc = sub {
 859     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 860     my $d = {
 861         type => $type,
 862         format_description => $unit,
 863         description => "Maximum $what in $longunit.",
 864         optional => 1,
 865     };
 866     $d->{minimum} = $minimum if defined($minimum);
 867     $drivedesc_base{$key} = $d;
 868 };
 869 # throughput: (leaky bucket)
 870 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 871 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 872 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 873 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 874 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 875 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 876 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 877 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 878 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 879
 880 # pools: (pool of IO before throttling starts taking effect)
 881 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 882 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 883 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 884 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 885 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 886 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 887
 888 # burst lengths
 889 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 890 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 891 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 892 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 893 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 894 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 895
 896 # legacy support
 897 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
 898 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
 899 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
 900 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
 901
 902 my $ide_fmt = {
 903     %drivedesc_base,
 904     %model_fmt,
 905 };
 906 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 907
 908 my $idedesc = {
 909     optional => 1,
 910     type => 'string', format => $ide_fmt,
 911     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 912 };
 913 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 914
 915 my $scsi_fmt = {
 916     %drivedesc_base,
 917     %iothread_fmt,
 918     %queues_fmt,
 919     %scsiblock_fmt,
 920 };
 921 my $scsidesc = {
 922     optional => 1,
 923     type => 'string', format => $scsi_fmt,
 924     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 925 };
 926 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 927
 928 my $sata_fmt = {
 929     %drivedesc_base,
 930 };
 931 my $satadesc = {
 932     optional => 1,
 933     type => 'string', format => $sata_fmt,
 934     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 935 };
 936 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 937
 938 my $virtio_fmt = {
 939     %drivedesc_base,
 940     %iothread_fmt,
 941 };
 942 my $virtiodesc = {
 943     optional => 1,
 944     type => 'string', format => $virtio_fmt,
 945     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 946 };
 947 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 948
 949 my $alldrive_fmt = {
 950     %drivedesc_base,
 951     %iothread_fmt,
 952     %model_fmt,
 953     %queues_fmt,
 954     %scsiblock_fmt,
 955 };
 956
 957 my $efidisk_fmt = {
 958     volume => { alias => 'file' },
 959     file => {
 960         type => 'string',
 961         format => 'pve-volume-id-or-qm-path',
 962         default_key => 1,
 963         format_description => 'volume',
 964         description => "The drive's backing volume.",
 965     },
 966     format => get_standard_option('pve-qm-image-format'),
 967     size => {
 968         type => 'string',
 969         format => 'disk-size',
 970         format_description => 'DiskSize',
 971         description => "Disk size. This is purely informational and has no effect.",
 972         optional => 1,
 973     },
 974 };
 975
 976 my $efidisk_desc = {
 977     optional => 1,
 978     type => 'string', format => $efidisk_fmt,
 979     description => "Configure a Disk for storing EFI vars",
 980 };
 981
 982 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 983
 984 my $usb_fmt = {
 985     host => {
 986         default_key => 1,
 987         type => 'string', format => 'pve-qm-usb-device',
 988         format_description => 'HOSTUSBDEVICE|spice',
 989         description => <<EODESCR,
 990 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
 991
 992  'bus-port(.port)*' (decimal numbers) or
 993  'vendor_id:product_id' (hexadeciaml numbers) or
 994  'spice'
 995
 996 You can use the 'lsusb -t' command to list existing usb devices.
 997
 998 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
 999
1000 The value 'spice' can be used to add a usb redirection devices for spice.
1001 EODESCR
1002     },
1003     usb3 => {
1004         optional => 1,
1005         type => 'boolean',
1006         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1007         default => 0,
1008     },
1009 };
1010
1011 my $usbdesc = {
1012     optional => 1,
1013     type => 'string', format => $usb_fmt,
1014     description => "Configure an USB device (n is 0 to 4).",
1015 };
1016 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1017
1018 # NOTE: the match-groups of this regex are used in parse_hostpci
1019 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1020 my $hostpci_fmt = {
1021     host => {
1022         default_key => 1,
1023         type => 'string',
1024         pattern => qr/$PCIRE(;$PCIRE)*/,
1025         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1026         description => <<EODESCR,
1027 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1028 of PCI virtual functions of the host. HOSTPCIID syntax is:
1029
1030 'bus:dev.func' (hexadecimal numbers)
1031
1032 You can us the 'lspci' command to list existing PCI devices.
1033 EODESCR
1034     },
1035     rombar => {
1036         type => 'boolean',
1037         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1038         optional => 1,
1039         default => 1,
1040     },
1041     romfile => {
1042         type => 'string',
1043         pattern => '[^,;]+',
1044         format_description => 'string',
1045         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1046         optional => 1,
1047     },
1048     pcie => {
1049         type => 'boolean',
1050         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1051         optional => 1,
1052         default => 0,
1053     },
1054     'x-vga' => {
1055         type => 'boolean',
1056         description =>  "Enable vfio-vga device support.",
1057         optional => 1,
1058         default => 0,
1059     },
1060 };
1061 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1062
1063 my $hostpcidesc = {
1064         optional => 1,
1065         type => 'string', format => 'pve-qm-hostpci',
1066         description => "Map host PCI devices into guest.",
1067         verbose_description =>  <<EODESCR,
1068 Map host PCI devices into guest.
1069
1070 NOTE: This option allows direct access to host hardware. So it is no longer
1071 possible to migrate such machines - use with special care.
1072
1073 CAUTION: Experimental! User reported problems with this option.
1074 EODESCR
1075 };
1076 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1077
1078 my $serialdesc = {
1079         optional => 1,
1080         type => 'string',
1081         pattern => '(/dev/.+|socket)',
1082         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1083         verbose_description =>  <<EODESCR,
1084 Create a serial device inside the VM (n is 0 to 3), and pass through a
1085 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1086 host side (use 'qm terminal' to open a terminal connection).
1087
1088 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1089
1090 CAUTION: Experimental! User reported problems with this option.
1091 EODESCR
1092 };
1093
1094 my $paralleldesc= {
1095         optional => 1,
1096         type => 'string',
1097         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1098         description =>  "Map host parallel devices (n is 0 to 2).",
1099         verbose_description =>  <<EODESCR,
1100 Map host parallel devices (n is 0 to 2).
1101
1102 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1103
1104 CAUTION: Experimental! User reported problems with this option.
1105 EODESCR
1106 };
1107
1108 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1109     $confdesc->{"parallel$i"} = $paralleldesc;
1110 }
1111
1112 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1113     $confdesc->{"serial$i"} = $serialdesc;
1114 }
1115
1116 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1117     $confdesc->{"hostpci$i"} = $hostpcidesc;
1118 }
1119
1120 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1121     $drivename_hash->{"ide$i"} = 1;
1122     $confdesc->{"ide$i"} = $idedesc;
1123 }
1124
1125 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1126     $drivename_hash->{"sata$i"} = 1;
1127     $confdesc->{"sata$i"} = $satadesc;
1128 }
1129
1130 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1131     $drivename_hash->{"scsi$i"} = 1;
1132     $confdesc->{"scsi$i"} = $scsidesc ;
1133 }
1134
1135 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1136     $drivename_hash->{"virtio$i"} = 1;
1137     $confdesc->{"virtio$i"} = $virtiodesc;
1138 }
1139
1140 $drivename_hash->{efidisk0} = 1;
1141 $confdesc->{efidisk0} = $efidisk_desc;
1142
1143 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1144     $confdesc->{"usb$i"} = $usbdesc;
1145 }
1146
1147 my $unuseddesc = {
1148     optional => 1,
1149     type => 'string', format => 'pve-volume-id',
1150     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1151 };
1152
1153 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1154     $confdesc->{"unused$i"} = $unuseddesc;
1155 }
1156
1157 my $kvm_api_version = 0;
1158
1159 sub kvm_version {
1160
1161     return $kvm_api_version if $kvm_api_version;
1162
1163     my $fh = IO::File->new("</dev/kvm") ||
1164         return 0;
1165
1166     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1167         $kvm_api_version = $v;
1168     }
1169
1170     $fh->close();
1171
1172     return  $kvm_api_version;
1173 }
1174
1175 my $kvm_user_version;
1176
1177 sub kvm_user_version {
1178
1179     return $kvm_user_version if $kvm_user_version;
1180
1181     $kvm_user_version = 'unknown';
1182
1183     my $code = sub {
1184         my $line = shift;
1185         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1186             $kvm_user_version = $2;
1187         }
1188     };
1189
1190     eval { run_command("kvm -version", outfunc => $code); };
1191     warn $@ if $@;
1192
1193     return $kvm_user_version;
1194
1195 }
1196
1197 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1198
1199 sub valid_drive_names {
1200     # order is important - used to autoselect boot disk
1201     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1202             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1203             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1204             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1205             'efidisk0');
1206 }
1207
1208 sub is_valid_drivename {
1209     my $dev = shift;
1210
1211     return defined($drivename_hash->{$dev});
1212 }
1213
1214 sub option_exists {
1215     my $key = shift;
1216     return defined($confdesc->{$key});
1217 }
1218
1219 sub nic_models {
1220     return $nic_model_list;
1221 }
1222
1223 sub os_list_description {
1224
1225     return {
1226         other => 'Other',
1227         wxp => 'Windows XP',
1228         w2k => 'Windows 2000',
1229         w2k3 =>, 'Windows 2003',
1230         w2k8 => 'Windows 2008',
1231         wvista => 'Windows Vista',
1232         win7 => 'Windows 7',
1233         win8 => 'Windows 8/2012',
1234         win10 => 'Windows 10/2016',
1235         l24 => 'Linux 2.4',
1236         l26 => 'Linux 2.6',
1237     };
1238 }
1239
1240 my $cdrom_path;
1241
1242 sub get_cdrom_path {
1243
1244     return  $cdrom_path if $cdrom_path;
1245
1246     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1247     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1248     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1249 }
1250
1251 sub get_iso_path {
1252     my ($storecfg, $vmid, $cdrom) = @_;
1253
1254     if ($cdrom eq 'cdrom') {
1255         return get_cdrom_path();
1256     } elsif ($cdrom eq 'none') {
1257         return '';
1258     } elsif ($cdrom =~ m|^/|) {
1259         return $cdrom;
1260     } else {
1261         return PVE::Storage::path($storecfg, $cdrom);
1262     }
1263 }
1264
1265 # try to convert old style file names to volume IDs
1266 sub filename_to_volume_id {
1267     my ($vmid, $file, $media) = @_;
1268
1269     if (!($file eq 'none' || $file eq 'cdrom' ||
1270           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1271
1272         return undef if $file =~ m|/|;
1273
1274         if ($media && $media eq 'cdrom') {
1275             $file = "local:iso/$file";
1276         } else {
1277             $file = "local:$vmid/$file";
1278         }
1279     }
1280
1281     return $file;
1282 }
1283
1284 sub verify_media_type {
1285     my ($opt, $vtype, $media) = @_;
1286
1287     return if !$media;
1288
1289     my $etype;
1290     if ($media eq 'disk') {
1291         $etype = 'images';
1292     } elsif ($media eq 'cdrom') {
1293         $etype = 'iso';
1294     } else {
1295         die "internal error";
1296     }
1297
1298     return if ($vtype eq $etype);
1299
1300     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1301 }
1302
1303 sub cleanup_drive_path {
1304     my ($opt, $storecfg, $drive) = @_;
1305
1306     # try to convert filesystem paths to volume IDs
1307
1308     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1309         ($drive->{file} !~ m|^/dev/.+|) &&
1310         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1311         ($drive->{file} !~ m/^\d+$/)) {
1312         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1313         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1314         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1315         verify_media_type($opt, $vtype, $drive->{media});
1316         $drive->{file} = $volid;
1317     }
1318
1319     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1320 }
1321
1322 sub parse_hotplug_features {
1323     my ($data) = @_;
1324
1325     my $res = {};
1326
1327     return $res if $data eq '0';
1328
1329     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1330
1331     foreach my $feature (PVE::Tools::split_list($data)) {
1332         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1333             $res->{$1} = 1;
1334         } else {
1335             die "invalid hotplug feature '$feature'\n";
1336         }
1337     }
1338     return $res;
1339 }
1340
1341 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1342 sub pve_verify_hotplug_features {
1343     my ($value, $noerr) = @_;
1344
1345     return $value if parse_hotplug_features($value);
1346
1347     return undef if $noerr;
1348
1349     die "unable to parse hotplug option\n";
1350 }
1351
1352 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1353 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1354 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1355 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1356 #        [,iothread=on][,serial=serial][,model=model]
1357
1358 sub parse_drive {
1359     my ($key, $data) = @_;
1360
1361     my ($interface, $index);
1362
1363     if ($key =~ m/^([^\d]+)(\d+)$/) {
1364         $interface = $1;
1365         $index = $2;
1366     } else {
1367         return undef;
1368     }
1369
1370     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1371                                      : $confdesc->{$key}->{format};
1372     if (!$desc) {
1373         warn "invalid drive key: $key\n";
1374         return undef;
1375     }
1376     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1377     return undef if !$res;
1378     $res->{interface} = $interface;
1379     $res->{index} = $index;
1380
1381     my $error = 0;
1382     foreach my $opt (qw(bps bps_rd bps_wr)) {
1383         if (my $bps = defined(delete $res->{$opt})) {
1384             if (defined($res->{"m$opt"})) {
1385                 warn "both $opt and m$opt specified\n";
1386                 ++$error;
1387                 next;
1388             }
1389             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1390         }
1391     }
1392
1393     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1394     for my $requirement (
1395         [mbps_max => 'mbps'],
1396         [mbps_rd_max => 'mbps_rd'],
1397         [mbps_wr_max => 'mbps_wr'],
1398         [miops_max => 'miops'],
1399         [miops_rd_max => 'miops_rd'],
1400         [miops_wr_max => 'miops_wr'],
1401         [bps_max_length => 'mbps_max'],
1402         [bps_rd_max_length => 'mbps_rd_max'],
1403         [bps_wr_max_length => 'mbps_wr_max'],
1404         [iops_max_length => 'iops_max'],
1405         [iops_rd_max_length => 'iops_rd_max'],
1406         [iops_wr_max_length => 'iops_wr_max']) {
1407         my ($option, $requires) = @$requirement;
1408         if ($res->{$option} && !$res->{$requires}) {
1409             warn "$option requires $requires\n";
1410             ++$error;
1411         }
1412     }
1413
1414     return undef if $error;
1415
1416     return undef if $res->{mbps_rd} && $res->{mbps};
1417     return undef if $res->{mbps_wr} && $res->{mbps};
1418     return undef if $res->{iops_rd} && $res->{iops};
1419     return undef if $res->{iops_wr} && $res->{iops};
1420
1421     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1422         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1423         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1424         return undef if $res->{interface} eq 'virtio';
1425     }
1426
1427     if (my $size = $res->{size}) {
1428         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1429     }
1430
1431     return $res;
1432 }
1433
1434 sub print_drive {
1435     my ($vmid, $drive) = @_;
1436     my $data = { %$drive };
1437     delete $data->{$_} for qw(index interface);
1438     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1439 }
1440
1441 sub scsi_inquiry {
1442     my($fh, $noerr) = @_;
1443
1444     my $SG_IO = 0x2285;
1445     my $SG_GET_VERSION_NUM = 0x2282;
1446
1447     my $versionbuf = "\x00" x 8;
1448     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1449     if (!$ret) {
1450         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1451         return undef;
1452     }
1453     my $version = unpack("I", $versionbuf);
1454     if ($version < 30000) {
1455         die "scsi generic interface too old\n"  if !$noerr;
1456         return undef;
1457     }
1458
1459     my $buf = "\x00" x 36;
1460     my $sensebuf = "\x00" x 8;
1461     my $cmd = pack("C x3 C x1", 0x12, 36);
1462
1463     # see /usr/include/scsi/sg.h
1464     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1465
1466     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1467                       length($sensebuf), 0, length($buf), $buf,
1468                       $cmd, $sensebuf, 6000);
1469
1470     $ret = ioctl($fh, $SG_IO, $packet);
1471     if (!$ret) {
1472         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1473         return undef;
1474     }
1475
1476     my @res = unpack($sg_io_hdr_t, $packet);
1477     if ($res[17] || $res[18]) {
1478         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1479         return undef;
1480     }
1481
1482     my $res = {};
1483     (my $byte0, my $byte1, $res->{vendor},
1484      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1485
1486     $res->{removable} = $byte1 & 128 ? 1 : 0;
1487     $res->{type} = $byte0 & 31;
1488
1489     return $res;
1490 }
1491
1492 sub path_is_scsi {
1493     my ($path) = @_;
1494
1495     my $fh = IO::File->new("+<$path") || return undef;
1496     my $res = scsi_inquiry($fh, 1);
1497     close($fh);
1498
1499     return $res;
1500 }
1501
1502 sub machine_type_is_q35 {
1503     my ($conf) = @_;
1504
1505     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1506 }
1507
1508 sub print_tabletdevice_full {
1509     my ($conf) = @_;
1510
1511     my $q35 = machine_type_is_q35($conf);
1512
1513     # we use uhci for old VMs because tablet driver was buggy in older qemu
1514     my $usbbus = $q35 ? "ehci" : "uhci";
1515
1516     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1517 }
1518
1519 sub print_drivedevice_full {
1520     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1521
1522     my $device = '';
1523     my $maxdev = 0;
1524
1525     if ($drive->{interface} eq 'virtio') {
1526         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1527         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1528         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1529     } elsif ($drive->{interface} eq 'scsi') {
1530
1531         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1532         my $unit = $drive->{index} % $maxdev;
1533         my $devicetype = 'hd';
1534         my $path = '';
1535         if (drive_is_cdrom($drive)) {
1536             $devicetype = 'cd';
1537         } else {
1538             if ($drive->{file} =~ m|^/|) {
1539                 $path = $drive->{file};
1540                 if (my $info = path_is_scsi($path)) {
1541                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1542                         $devicetype = 'block';
1543                     } elsif ($info->{type} == 1) { # tape
1544                         $devicetype = 'generic';
1545                     }
1546                 }
1547             } else {
1548                  $path = PVE::Storage::path($storecfg, $drive->{file});
1549             }
1550
1551             if($path =~ m/^iscsi\:\/\//){
1552                 $devicetype = 'generic';
1553             }
1554         }
1555
1556         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1557             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1558         } else {
1559             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1560         }
1561
1562     } elsif ($drive->{interface} eq 'ide'){
1563         $maxdev = 2;
1564         my $controller = int($drive->{index} / $maxdev);
1565         my $unit = $drive->{index} % $maxdev;
1566         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1567
1568         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1569         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1570             $model = URI::Escape::uri_unescape($model);
1571             $device .= ",model=$model";
1572         }
1573     } elsif ($drive->{interface} eq 'sata'){
1574         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1575         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1576         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1577     } elsif ($drive->{interface} eq 'usb') {
1578         die "implement me";
1579         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1580     } else {
1581         die "unsupported interface type";
1582     }
1583
1584     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1585
1586     return $device;
1587 }
1588
1589 sub get_initiator_name {
1590     my $initiator;
1591
1592     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1593     while (defined(my $line = <$fh>)) {
1594         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1595         $initiator = $1;
1596         last;
1597     }
1598     $fh->close();
1599
1600     return $initiator;
1601 }
1602
1603 sub print_drive_full {
1604     my ($storecfg, $vmid, $drive) = @_;
1605
1606     my $path;
1607     my $volid = $drive->{file};
1608     my $format;
1609
1610     if (drive_is_cdrom($drive)) {
1611         $path = get_iso_path($storecfg, $vmid, $volid);
1612     } else {
1613         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1614         if ($storeid) {
1615             $path = PVE::Storage::path($storecfg, $volid);
1616             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1617             $format = qemu_img_format($scfg, $volname);
1618         } else {
1619             $path = $volid;
1620             $format = "raw";
1621         }
1622    }
1623
1624     my $opts = '';
1625     my @qemu_drive_options = qw(heads secs cyls trans media format cache snapshot rerror werror aio discard);
1626     foreach my $o (@qemu_drive_options) {
1627         $opts .= ",$o=$drive->{$o}" if $drive->{$o};
1628     }
1629     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1630         my ($dir, $qmpname) = @$type;
1631         if (my $v = $drive->{"mbps$dir"}) {
1632             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1633         }
1634         if (my $v = $drive->{"mbps${dir}_max"}) {
1635             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1636         }
1637         if (my $v = $drive->{"bps${dir}_max_length"}) {
1638             $opts .= ",throttling.bps$qmpname-max-length=$v";
1639         }
1640         if (my $v = $drive->{"iops${dir}"}) {
1641             $opts .= ",throttling.iops$qmpname=$v";
1642         }
1643         if (my $v = $drive->{"iops${dir}_max"}) {
1644             $opts .= ",throttling.iops$qmpname-max=$v";
1645         }
1646         if (my $v = $drive->{"iops${dir}_max_length"}) {
1647             $opts .= ",throttling.iops$qmpname-max-length=$v";
1648         }
1649     }
1650
1651     if (my $serial = $drive->{serial}) {
1652         $serial = URI::Escape::uri_unescape($serial);
1653         $opts .= ",serial=$serial";
1654     }
1655
1656     $opts .= ",format=$format" if $format && !$drive->{format};
1657
1658     my $cache_direct = 0;
1659
1660     if (my $cache = $drive->{cache}) {
1661         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1662     } elsif (!drive_is_cdrom($drive)) {
1663         $opts .= ",cache=none";
1664         $cache_direct = 1;
1665     }
1666
1667     # aio native works only with O_DIRECT
1668     if (!$drive->{aio}) {
1669         if($cache_direct) {
1670             $opts .= ",aio=native";
1671         } else {
1672             $opts .= ",aio=threads";
1673         }
1674     }
1675
1676     if (!drive_is_cdrom($drive)) {
1677         my $detectzeroes;
1678         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1679             $detectzeroes = 'off';
1680         } elsif ($drive->{discard}) {
1681             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1682         } else {
1683             # This used to be our default with discard not being specified:
1684             $detectzeroes = 'on';
1685         }
1686         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1687     }
1688
1689     my $pathinfo = $path ? "file=$path," : '';
1690
1691     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1692 }
1693
1694 sub print_netdevice_full {
1695     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1696
1697     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1698
1699     my $device = $net->{model};
1700     if ($net->{model} eq 'virtio') {
1701          $device = 'virtio-net-pci';
1702      };
1703
1704     my $pciaddr = print_pci_addr("$netid", $bridges);
1705     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1706     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1707         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1708         my $vectors = $net->{queues} * 2 + 2;
1709         $tmpstr .= ",vectors=$vectors,mq=on";
1710     }
1711     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1712
1713     if ($use_old_bios_files) {
1714         my $romfile;
1715         if ($device eq 'virtio-net-pci') {
1716             $romfile = 'pxe-virtio.rom';
1717         } elsif ($device eq 'e1000') {
1718             $romfile = 'pxe-e1000.rom';
1719         } elsif ($device eq 'ne2k') {
1720             $romfile = 'pxe-ne2k_pci.rom';
1721         } elsif ($device eq 'pcnet') {
1722             $romfile = 'pxe-pcnet.rom';
1723         } elsif ($device eq 'rtl8139') {
1724             $romfile = 'pxe-rtl8139.rom';
1725         }
1726         $tmpstr .= ",romfile=$romfile" if $romfile;
1727     }
1728
1729     return $tmpstr;
1730 }
1731
1732 sub print_netdev_full {
1733     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1734
1735     my $i = '';
1736     if ($netid =~ m/^net(\d+)$/) {
1737         $i = int($1);
1738     }
1739
1740     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1741
1742     my $ifname = "tap${vmid}i$i";
1743
1744     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1745     die "interface name '$ifname' is too long (max 15 character)\n"
1746         if length($ifname) >= 16;
1747
1748     my $vhostparam = '';
1749     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1750
1751     my $vmname = $conf->{name} || "vm$vmid";
1752
1753     my $netdev = "";
1754     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1755
1756     if ($net->{bridge}) {
1757         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1758     } else {
1759         $netdev = "type=user,id=$netid,hostname=$vmname";
1760     }
1761
1762     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1763
1764     return $netdev;
1765 }
1766
1767
1768 sub print_cpu_device {
1769     my ($conf, $id) = @_;
1770
1771     my $kvm = $conf->{kvm} // 1;
1772     my $cpu = $kvm ? "kvm64" : "qemu64";
1773     if (my $cputype = $conf->{cpu}) {
1774         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1775             or die "Cannot parse cpu description: $cputype\n";
1776         $cpu = $cpuconf->{cputype};
1777     }
1778
1779     my $cores = $conf->{cores} || 1;
1780
1781     my $current_core = ($id - 1) % $cores;
1782     my $current_socket = int(($id - 1 - $current_core)/$cores);
1783
1784     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1785 }
1786
1787 sub drive_is_cdrom {
1788     my ($drive) = @_;
1789
1790     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1791
1792 }
1793
1794 sub parse_number_sets {
1795     my ($set) = @_;
1796     my $res = [];
1797     foreach my $part (split(/;/, $set)) {
1798         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1799             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1800             push @$res, [ $1, $2 ];
1801         } else {
1802             die "invalid range: $part\n";
1803         }
1804     }
1805     return $res;
1806 }
1807
1808 sub parse_numa {
1809     my ($data) = @_;
1810
1811     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1812     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1813     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1814     return $res;
1815 }
1816
1817 sub parse_hostpci {
1818     my ($value) = @_;
1819
1820     return undef if !$value;
1821
1822     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1823
1824     my @idlist = split(/;/, $res->{host});
1825     delete $res->{host};
1826     foreach my $id (@idlist) {
1827         if ($id =~ /^$PCIRE$/) {
1828             if (defined($2)) {
1829                 push @{$res->{pciid}}, { id => $1, function => $2 };
1830             } else {
1831                 my $pcidevices = lspci($1);
1832                 $res->{pciid} = $pcidevices->{$1};
1833             }
1834         } else {
1835             # should have been caught by parse_property_string already
1836             die "failed to parse PCI id: $id\n";
1837         }
1838     }
1839     return $res;
1840 }
1841
1842 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1843 sub parse_net {
1844     my ($data) = @_;
1845
1846     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1847     if ($@) {
1848         warn $@;
1849         return undef;
1850     }
1851     if (!defined($res->{macaddr})) {
1852         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1853         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1854     }
1855     return $res;
1856 }
1857
1858 sub print_net {
1859     my $net = shift;
1860
1861     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1862 }
1863
1864 sub add_random_macs {
1865     my ($settings) = @_;
1866
1867     foreach my $opt (keys %$settings) {
1868         next if $opt !~ m/^net(\d+)$/;
1869         my $net = parse_net($settings->{$opt});
1870         next if !$net;
1871         $settings->{$opt} = print_net($net);
1872     }
1873 }
1874
1875 sub vm_is_volid_owner {
1876     my ($storecfg, $vmid, $volid) = @_;
1877
1878     if ($volid !~  m|^/|) {
1879         my ($path, $owner);
1880         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1881         if ($owner && ($owner == $vmid)) {
1882             return 1;
1883         }
1884     }
1885
1886     return undef;
1887 }
1888
1889 sub split_flagged_list {
1890     my $text = shift || '';
1891     $text =~ s/[,;]/ /g;
1892     $text =~ s/^\s+//;
1893     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1894 }
1895
1896 sub join_flagged_list {
1897     my ($how, $lst) = @_;
1898     join $how, map { $lst->{$_} . $_ } keys %$lst;
1899 }
1900
1901 sub vmconfig_delete_pending_option {
1902     my ($conf, $key, $force) = @_;
1903
1904     delete $conf->{pending}->{$key};
1905     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1906     $pending_delete_hash->{$key} = $force ? '!' : '';
1907     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1908 }
1909
1910 sub vmconfig_undelete_pending_option {
1911     my ($conf, $key) = @_;
1912
1913     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1914     delete $pending_delete_hash->{$key};
1915
1916     if (%$pending_delete_hash) {
1917         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1918     } else {
1919         delete $conf->{pending}->{delete};
1920     }
1921 }
1922
1923 sub vmconfig_register_unused_drive {
1924     my ($storecfg, $vmid, $conf, $drive) = @_;
1925
1926     if (!drive_is_cdrom($drive)) {
1927         my $volid = $drive->{file};
1928         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1929             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1930         }
1931     }
1932 }
1933
1934 sub vmconfig_cleanup_pending {
1935     my ($conf) = @_;
1936
1937     # remove pending changes when nothing changed
1938     my $changes;
1939     foreach my $opt (keys %{$conf->{pending}}) {
1940         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1941             $changes = 1;
1942             delete $conf->{pending}->{$opt};
1943         }
1944     }
1945
1946     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1947     my $pending_delete_hash = {};
1948     while (my ($opt, $force) = each %$current_delete_hash) {
1949         if (defined($conf->{$opt})) {
1950             $pending_delete_hash->{$opt} = $force;
1951         } else {
1952             $changes = 1;
1953         }
1954     }
1955
1956     if (%$pending_delete_hash) {
1957         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1958     } else {
1959         delete $conf->{pending}->{delete};
1960     }
1961
1962     return $changes;
1963 }
1964
1965 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1966 my $smbios1_fmt = {
1967     uuid => {
1968         type => 'string',
1969         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1970         format_description => 'UUID',
1971         description => "Set SMBIOS1 UUID.",
1972         optional => 1,
1973     },
1974     version => {
1975         type => 'string',
1976         pattern => '\S+',
1977         format_description => 'string',
1978         description => "Set SMBIOS1 version.",
1979         optional => 1,
1980     },
1981     serial => {
1982         type => 'string',
1983         pattern => '\S+',
1984         format_description => 'string',
1985         description => "Set SMBIOS1 serial number.",
1986         optional => 1,
1987     },
1988     manufacturer => {
1989         type => 'string',
1990         pattern => '\S+',
1991         format_description => 'string',
1992         description => "Set SMBIOS1 manufacturer.",
1993         optional => 1,
1994     },
1995     product => {
1996         type => 'string',
1997         pattern => '\S+',
1998         format_description => 'string',
1999         description => "Set SMBIOS1 product ID.",
2000         optional => 1,
2001     },
2002     sku => {
2003         type => 'string',
2004         pattern => '\S+',
2005         format_description => 'string',
2006         description => "Set SMBIOS1 SKU string.",
2007         optional => 1,
2008     },
2009     family => {
2010         type => 'string',
2011         pattern => '\S+',
2012         format_description => 'string',
2013         description => "Set SMBIOS1 family string.",
2014         optional => 1,
2015     },
2016 };
2017
2018 sub parse_smbios1 {
2019     my ($data) = @_;
2020
2021     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2022     warn $@ if $@;
2023     return $res;
2024 }
2025
2026 sub print_smbios1 {
2027     my ($smbios1) = @_;
2028     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2029 }
2030
2031 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2032
2033 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2034 sub verify_bootdisk {
2035     my ($value, $noerr) = @_;
2036
2037     return $value if is_valid_drivename($value);
2038
2039     return undef if $noerr;
2040
2041     die "invalid boot disk '$value'\n";
2042 }
2043
2044 sub parse_watchdog {
2045     my ($value) = @_;
2046
2047     return undef if !$value;
2048
2049     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2050     warn $@ if $@;
2051     return $res;
2052 }
2053
2054 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2055 sub verify_usb_device {
2056     my ($value, $noerr) = @_;
2057
2058     return $value if parse_usb_device($value);
2059
2060     return undef if $noerr;
2061
2062     die "unable to parse usb device\n";
2063 }
2064
2065 # add JSON properties for create and set function
2066 sub json_config_properties {
2067     my $prop = shift;
2068
2069     foreach my $opt (keys %$confdesc) {
2070         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2071         $prop->{$opt} = $confdesc->{$opt};
2072     }
2073
2074     return $prop;
2075 }
2076
2077 sub check_type {
2078     my ($key, $value) = @_;
2079
2080     die "unknown setting '$key'\n" if !$confdesc->{$key};
2081
2082     my $type = $confdesc->{$key}->{type};
2083
2084     if (!defined($value)) {
2085         die "got undefined value\n";
2086     }
2087
2088     if ($value =~ m/[\n\r]/) {
2089         die "property contains a line feed\n";
2090     }
2091
2092     if ($type eq 'boolean') {
2093         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2094         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2095         die "type check ('boolean') failed - got '$value'\n";
2096     } elsif ($type eq 'integer') {
2097         return int($1) if $value =~ m/^(\d+)$/;
2098         die "type check ('integer') failed - got '$value'\n";
2099     } elsif ($type eq 'number') {
2100         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2101         die "type check ('number') failed - got '$value'\n";
2102     } elsif ($type eq 'string') {
2103         if (my $fmt = $confdesc->{$key}->{format}) {
2104             PVE::JSONSchema::check_format($fmt, $value);
2105             return $value;
2106         }
2107         $value =~ s/^\"(.*)\"$/$1/;
2108         return $value;
2109     } else {
2110         die "internal error"
2111     }
2112 }
2113
2114 sub check_iommu_support{
2115     #fixme : need to check IOMMU support
2116     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2117
2118     my $iommu=1;
2119     return $iommu;
2120
2121 }
2122
2123 sub touch_config {
2124     my ($vmid) = @_;
2125
2126     my $conf = PVE::QemuConfig->config_file($vmid);
2127     utime undef, undef, $conf;
2128 }
2129
2130 sub destroy_vm {
2131     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2132
2133     my $conffile = PVE::QemuConfig->config_file($vmid);
2134
2135     my $conf = PVE::QemuConfig->load_config($vmid);
2136
2137     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2138
2139     if ($conf->{template}) {
2140         # check if any base image is still used by a linked clone
2141         foreach_drive($conf, sub {
2142                 my ($ds, $drive) = @_;
2143
2144                 return if drive_is_cdrom($drive);
2145
2146                 my $volid = $drive->{file};
2147
2148                 return if !$volid || $volid =~ m|^/|;
2149
2150                 die "base volume '$volid' is still in use by linked cloned\n"
2151                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2152
2153         });
2154     }
2155
2156     # only remove disks owned by this VM
2157     foreach_drive($conf, sub {
2158         my ($ds, $drive) = @_;
2159
2160         return if drive_is_cdrom($drive);
2161
2162         my $volid = $drive->{file};
2163
2164         return if !$volid || $volid =~ m|^/|;
2165
2166         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2167         return if !$path || !$owner || ($owner != $vmid);
2168
2169         eval {
2170             PVE::Storage::vdisk_free($storecfg, $volid);
2171         };
2172         warn "Could not remove disk '$volid', check manually: $@" if $@;
2173
2174     });
2175
2176     if ($keep_empty_config) {
2177         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2178     } else {
2179         unlink $conffile;
2180     }
2181
2182     # also remove unused disk
2183     eval {
2184         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2185
2186         eval {
2187             PVE::Storage::foreach_volid($dl, sub {
2188                 my ($volid, $sid, $volname, $d) = @_;
2189                 PVE::Storage::vdisk_free($storecfg, $volid);
2190             });
2191         };
2192         warn $@ if $@;
2193
2194     };
2195     warn $@ if $@;
2196 }
2197
2198 sub parse_vm_config {
2199     my ($filename, $raw) = @_;
2200
2201     return undef if !defined($raw);
2202
2203     my $res = {
2204         digest => Digest::SHA::sha1_hex($raw),
2205         snapshots => {},
2206         pending => {},
2207     };
2208
2209     $filename =~ m|/qemu-server/(\d+)\.conf$|
2210         || die "got strange filename '$filename'";
2211
2212     my $vmid = $1;
2213
2214     my $conf = $res;
2215     my $descr;
2216     my $section = '';
2217
2218     my @lines = split(/\n/, $raw);
2219     foreach my $line (@lines) {
2220         next if $line =~ m/^\s*$/;
2221
2222         if ($line =~ m/^\[PENDING\]\s*$/i) {
2223             $section = 'pending';
2224             if (defined($descr)) {
2225                 $descr =~ s/\s+$//;
2226                 $conf->{description} = $descr;
2227             }
2228             $descr = undef;
2229             $conf = $res->{$section} = {};
2230             next;
2231
2232         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2233             $section = $1;
2234             if (defined($descr)) {
2235                 $descr =~ s/\s+$//;
2236                 $conf->{description} = $descr;
2237             }
2238             $descr = undef;
2239             $conf = $res->{snapshots}->{$section} = {};
2240             next;
2241         }
2242
2243         if ($line =~ m/^\#(.*)\s*$/) {
2244             $descr = '' if !defined($descr);
2245             $descr .= PVE::Tools::decode_text($1) . "\n";
2246             next;
2247         }
2248
2249         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2250             $descr = '' if !defined($descr);
2251             $descr .= PVE::Tools::decode_text($2);
2252         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2253             $conf->{snapstate} = $1;
2254         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2255             my $key = $1;
2256             my $value = $2;
2257             $conf->{$key} = $value;
2258         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2259             my $value = $1;
2260             if ($section eq 'pending') {
2261                 $conf->{delete} = $value; # we parse this later
2262             } else {
2263                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2264             }
2265         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2266             my $key = $1;
2267             my $value = $2;
2268             eval { $value = check_type($key, $value); };
2269             if ($@) {
2270                 warn "vm $vmid - unable to parse value of '$key' - $@";
2271             } else {
2272                 $key = 'ide2' if $key eq 'cdrom';
2273                 my $fmt = $confdesc->{$key}->{format};
2274                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2275                     my $v = parse_drive($key, $value);
2276                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2277                         $v->{file} = $volid;
2278                         $value = print_drive($vmid, $v);
2279                     } else {
2280                         warn "vm $vmid - unable to parse value of '$key'\n";
2281                         next;
2282                     }
2283                 }
2284
2285                 $conf->{$key} = $value;
2286             }
2287         }
2288     }
2289
2290     if (defined($descr)) {
2291         $descr =~ s/\s+$//;
2292         $conf->{description} = $descr;
2293     }
2294     delete $res->{snapstate}; # just to be sure
2295
2296     return $res;
2297 }
2298
2299 sub write_vm_config {
2300     my ($filename, $conf) = @_;
2301
2302     delete $conf->{snapstate}; # just to be sure
2303
2304     if ($conf->{cdrom}) {
2305         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2306         $conf->{ide2} = $conf->{cdrom};
2307         delete $conf->{cdrom};
2308     }
2309
2310     # we do not use 'smp' any longer
2311     if ($conf->{sockets}) {
2312         delete $conf->{smp};
2313     } elsif ($conf->{smp}) {
2314         $conf->{sockets} = $conf->{smp};
2315         delete $conf->{cores};
2316         delete $conf->{smp};
2317     }
2318
2319     my $used_volids = {};
2320
2321     my $cleanup_config = sub {
2322         my ($cref, $pending, $snapname) = @_;
2323
2324         foreach my $key (keys %$cref) {
2325             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2326                 $key eq 'snapstate' || $key eq 'pending';
2327             my $value = $cref->{$key};
2328             if ($key eq 'delete') {
2329                 die "propertry 'delete' is only allowed in [PENDING]\n"
2330                     if !$pending;
2331                 # fixme: check syntax?
2332                 next;
2333             }
2334             eval { $value = check_type($key, $value); };
2335             die "unable to parse value of '$key' - $@" if $@;
2336
2337             $cref->{$key} = $value;
2338
2339             if (!$snapname && is_valid_drivename($key)) {
2340                 my $drive = parse_drive($key, $value);
2341                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2342             }
2343         }
2344     };
2345
2346     &$cleanup_config($conf);
2347
2348     &$cleanup_config($conf->{pending}, 1);
2349
2350     foreach my $snapname (keys %{$conf->{snapshots}}) {
2351         die "internal error" if $snapname eq 'pending';
2352         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2353     }
2354
2355     # remove 'unusedX' settings if we re-add a volume
2356     foreach my $key (keys %$conf) {
2357         my $value = $conf->{$key};
2358         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2359             delete $conf->{$key};
2360         }
2361     }
2362
2363     my $generate_raw_config = sub {
2364         my ($conf, $pending) = @_;
2365
2366         my $raw = '';
2367
2368         # add description as comment to top of file
2369         if (defined(my $descr = $conf->{description})) {
2370             if ($descr) {
2371                 foreach my $cl (split(/\n/, $descr)) {
2372                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2373                 }
2374             } else {
2375                 $raw .= "#\n" if $pending;
2376             }
2377         }
2378
2379         foreach my $key (sort keys %$conf) {
2380             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2381             $raw .= "$key: $conf->{$key}\n";
2382         }
2383         return $raw;
2384     };
2385
2386     my $raw = &$generate_raw_config($conf);
2387
2388     if (scalar(keys %{$conf->{pending}})){
2389         $raw .= "\n[PENDING]\n";
2390         $raw .= &$generate_raw_config($conf->{pending}, 1);
2391     }
2392
2393     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2394         $raw .= "\n[$snapname]\n";
2395         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2396     }
2397
2398     return $raw;
2399 }
2400
2401 sub load_defaults {
2402
2403     my $res = {};
2404
2405     # we use static defaults from our JSON schema configuration
2406     foreach my $key (keys %$confdesc) {
2407         if (defined(my $default = $confdesc->{$key}->{default})) {
2408             $res->{$key} = $default;
2409         }
2410     }
2411
2412     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2413     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2414
2415     return $res;
2416 }
2417
2418 sub config_list {
2419     my $vmlist = PVE::Cluster::get_vmlist();
2420     my $res = {};
2421     return $res if !$vmlist || !$vmlist->{ids};
2422     my $ids = $vmlist->{ids};
2423
2424     foreach my $vmid (keys %$ids) {
2425         my $d = $ids->{$vmid};
2426         next if !$d->{node} || $d->{node} ne $nodename;
2427         next if !$d->{type} || $d->{type} ne 'qemu';
2428         $res->{$vmid}->{exists} = 1;
2429     }
2430     return $res;
2431 }
2432
2433 # test if VM uses local resources (to prevent migration)
2434 sub check_local_resources {
2435     my ($conf, $noerr) = @_;
2436
2437     my $loc_res = 0;
2438
2439     $loc_res = 1 if $conf->{hostusb}; # old syntax
2440     $loc_res = 1 if $conf->{hostpci}; # old syntax
2441
2442     foreach my $k (keys %$conf) {
2443         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2444         # sockets are safe: they will recreated be on the target side post-migrate
2445         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2446         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2447     }
2448
2449     die "VM uses local resources\n" if $loc_res && !$noerr;
2450
2451     return $loc_res;
2452 }
2453
2454 # check if used storages are available on all nodes (use by migrate)
2455 sub check_storage_availability {
2456     my ($storecfg, $conf, $node) = @_;
2457
2458     foreach_drive($conf, sub {
2459         my ($ds, $drive) = @_;
2460
2461         my $volid = $drive->{file};
2462         return if !$volid;
2463
2464         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2465         return if !$sid;
2466
2467         # check if storage is available on both nodes
2468         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2469         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2470    });
2471 }
2472
2473 # list nodes where all VM images are available (used by has_feature API)
2474 sub shared_nodes {
2475     my ($conf, $storecfg) = @_;
2476
2477     my $nodelist = PVE::Cluster::get_nodelist();
2478     my $nodehash = { map { $_ => 1 } @$nodelist };
2479     my $nodename = PVE::INotify::nodename();
2480
2481     foreach_drive($conf, sub {
2482         my ($ds, $drive) = @_;
2483
2484         my $volid = $drive->{file};
2485         return if !$volid;
2486
2487         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2488         if ($storeid) {
2489             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2490             if ($scfg->{disable}) {
2491                 $nodehash = {};
2492             } elsif (my $avail = $scfg->{nodes}) {
2493                 foreach my $node (keys %$nodehash) {
2494                     delete $nodehash->{$node} if !$avail->{$node};
2495                 }
2496             } elsif (!$scfg->{shared}) {
2497                 foreach my $node (keys %$nodehash) {
2498                     delete $nodehash->{$node} if $node ne $nodename
2499                 }
2500             }
2501         }
2502     });
2503
2504     return $nodehash
2505 }
2506
2507 sub check_cmdline {
2508     my ($pidfile, $pid) = @_;
2509
2510     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2511     if (defined($fh)) {
2512         my $line = <$fh>;
2513         $fh->close;
2514         return undef if !$line;
2515         my @param = split(/\0/, $line);
2516
2517         my $cmd = $param[0];
2518         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2519
2520         for (my $i = 0; $i < scalar (@param); $i++) {
2521             my $p = $param[$i];
2522             next if !$p;
2523             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2524                 my $p = $param[$i+1];
2525                 return 1 if $p && ($p eq $pidfile);
2526                 return undef;
2527             }
2528         }
2529     }
2530     return undef;
2531 }
2532
2533 sub check_running {
2534     my ($vmid, $nocheck, $node) = @_;
2535
2536     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2537
2538     die "unable to find configuration file for VM $vmid - no such machine\n"
2539         if !$nocheck && ! -f $filename;
2540
2541     my $pidfile = pidfile_name($vmid);
2542
2543     if (my $fd = IO::File->new("<$pidfile")) {
2544         my $st = stat($fd);
2545         my $line = <$fd>;
2546         close($fd);
2547
2548         my $mtime = $st->mtime;
2549         if ($mtime > time()) {
2550             warn "file '$filename' modified in future\n";
2551         }
2552
2553         if ($line =~ m/^(\d+)$/) {
2554             my $pid = $1;
2555             if (check_cmdline($pidfile, $pid)) {
2556                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2557                     return $pid;
2558                 }
2559             }
2560         }
2561     }
2562
2563     return undef;
2564 }
2565
2566 sub vzlist {
2567
2568     my $vzlist = config_list();
2569
2570     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2571
2572     while (defined(my $de = $fd->read)) {
2573         next if $de !~ m/^(\d+)\.pid$/;
2574         my $vmid = $1;
2575         next if !defined($vzlist->{$vmid});
2576         if (my $pid = check_running($vmid)) {
2577             $vzlist->{$vmid}->{pid} = $pid;
2578         }
2579     }
2580
2581     return $vzlist;
2582 }
2583
2584 sub disksize {
2585     my ($storecfg, $conf) = @_;
2586
2587     my $bootdisk = $conf->{bootdisk};
2588     return undef if !$bootdisk;
2589     return undef if !is_valid_drivename($bootdisk);
2590
2591     return undef if !$conf->{$bootdisk};
2592
2593     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2594     return undef if !defined($drive);
2595
2596     return undef if drive_is_cdrom($drive);
2597
2598     my $volid = $drive->{file};
2599     return undef if !$volid;
2600
2601     return $drive->{size};
2602 }
2603
2604 my $last_proc_pid_stat;
2605
2606 # get VM status information
2607 # This must be fast and should not block ($full == false)
2608 # We only query KVM using QMP if $full == true (this can be slow)
2609 sub vmstatus {
2610     my ($opt_vmid, $full) = @_;
2611
2612     my $res = {};
2613
2614     my $storecfg = PVE::Storage::config();
2615
2616     my $list = vzlist();
2617     my $defaults = load_defaults();
2618
2619     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2620
2621     my $cpucount = $cpuinfo->{cpus} || 1;
2622
2623     foreach my $vmid (keys %$list) {
2624         next if $opt_vmid && ($vmid ne $opt_vmid);
2625
2626         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2627         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2628
2629         my $d = {};
2630         $d->{pid} = $list->{$vmid}->{pid};
2631
2632         # fixme: better status?
2633         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2634
2635         my $size = disksize($storecfg, $conf);
2636         if (defined($size)) {
2637             $d->{disk} = 0; # no info available
2638             $d->{maxdisk} = $size;
2639         } else {
2640             $d->{disk} = 0;
2641             $d->{maxdisk} = 0;
2642         }
2643
2644         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2645             * ($conf->{cores} || $defaults->{cores});
2646         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2647         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2648
2649         $d->{name} = $conf->{name} || "VM $vmid";
2650         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2651             : $defaults->{memory}*(1024*1024);
2652
2653         if ($conf->{balloon}) {
2654             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2655             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2656                 : $defaults->{shares};
2657         }
2658
2659         $d->{uptime} = 0;
2660         $d->{cpu} = 0;
2661         $d->{mem} = 0;
2662
2663         $d->{netout} = 0;
2664         $d->{netin} = 0;
2665
2666         $d->{diskread} = 0;
2667         $d->{diskwrite} = 0;
2668
2669         $d->{template} = PVE::QemuConfig->is_template($conf);
2670
2671         $d->{serial} = 1 if conf_has_serial($conf);
2672
2673         $res->{$vmid} = $d;
2674     }
2675
2676     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2677     foreach my $dev (keys %$netdev) {
2678         next if $dev !~ m/^tap([1-9]\d*)i/;
2679         my $vmid = $1;
2680         my $d = $res->{$vmid};
2681         next if !$d;
2682
2683         $d->{netout} += $netdev->{$dev}->{receive};
2684         $d->{netin} += $netdev->{$dev}->{transmit};
2685
2686         if ($full) {
2687             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2688             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2689         }
2690
2691     }
2692
2693     my $ctime = gettimeofday;
2694
2695     foreach my $vmid (keys %$list) {
2696
2697         my $d = $res->{$vmid};
2698         my $pid = $d->{pid};
2699         next if !$pid;
2700
2701         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2702         next if !$pstat; # not running
2703
2704         my $used = $pstat->{utime} + $pstat->{stime};
2705
2706         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2707
2708         if ($pstat->{vsize}) {
2709             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2710         }
2711
2712         my $old = $last_proc_pid_stat->{$pid};
2713         if (!$old) {
2714             $last_proc_pid_stat->{$pid} = {
2715                 time => $ctime,
2716                 used => $used,
2717                 cpu => 0,
2718             };
2719             next;
2720         }
2721
2722         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2723
2724         if ($dtime > 1000) {
2725             my $dutime = $used -  $old->{used};
2726
2727             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2728             $last_proc_pid_stat->{$pid} = {
2729                 time => $ctime,
2730                 used => $used,
2731                 cpu => $d->{cpu},
2732             };
2733         } else {
2734             $d->{cpu} = $old->{cpu};
2735         }
2736     }
2737
2738     return $res if !$full;
2739
2740     my $qmpclient = PVE::QMPClient->new();
2741
2742     my $ballooncb = sub {
2743         my ($vmid, $resp) = @_;
2744
2745         my $info = $resp->{'return'};
2746         return if !$info->{max_mem};
2747
2748         my $d = $res->{$vmid};
2749
2750         # use memory assigned to VM
2751         $d->{maxmem} = $info->{max_mem};
2752         $d->{balloon} = $info->{actual};
2753
2754         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2755             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2756             $d->{freemem} = $info->{free_mem};
2757         }
2758
2759         $d->{ballooninfo} = $info;
2760     };
2761
2762     my $blockstatscb = sub {
2763         my ($vmid, $resp) = @_;
2764         my $data = $resp->{'return'} || [];
2765         my $totalrdbytes = 0;
2766         my $totalwrbytes = 0;
2767
2768         for my $blockstat (@$data) {
2769             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2770             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2771
2772             $blockstat->{device} =~ s/drive-//;
2773             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2774         }
2775         $res->{$vmid}->{diskread} = $totalrdbytes;
2776         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2777     };
2778
2779     my $statuscb = sub {
2780         my ($vmid, $resp) = @_;
2781
2782         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2783         # this fails if ballon driver is not loaded, so this must be
2784         # the last commnand (following command are aborted if this fails).
2785         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2786
2787         my $status = 'unknown';
2788         if (!defined($status = $resp->{'return'}->{status})) {
2789             warn "unable to get VM status\n";
2790             return;
2791         }
2792
2793         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2794     };
2795
2796     foreach my $vmid (keys %$list) {
2797         next if $opt_vmid && ($vmid ne $opt_vmid);
2798         next if !$res->{$vmid}->{pid}; # not running
2799         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2800     }
2801
2802     $qmpclient->queue_execute(undef, 2);
2803
2804     foreach my $vmid (keys %$list) {
2805         next if $opt_vmid && ($vmid ne $opt_vmid);
2806         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2807     }
2808
2809     return $res;
2810 }
2811
2812 sub foreach_drive {
2813     my ($conf, $func, @param) = @_;
2814
2815     foreach my $ds (valid_drive_names()) {
2816         next if !defined($conf->{$ds});
2817
2818         my $drive = parse_drive($ds, $conf->{$ds});
2819         next if !$drive;
2820
2821         &$func($ds, $drive, @param);
2822     }
2823 }
2824
2825 sub foreach_volid {
2826     my ($conf, $func, @param) = @_;
2827
2828     my $volhash = {};
2829
2830     my $test_volid = sub {
2831         my ($volid, $is_cdrom, $replicate, $snapname) = @_;
2832
2833         return if !$volid;
2834
2835         $volhash->{$volid}->{cdrom} //= 1;
2836         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2837
2838         $volhash->{$volid}->{replicate} //= 0;
2839         $volhash->{$volid}->{replicate} = 1 if $replicate;
2840
2841         $volhash->{$volid}->{referenced_in_config} //= 0;
2842         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
2843
2844         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
2845             if defined($snapname);
2846     };
2847
2848     foreach_drive($conf, sub {
2849         my ($ds, $drive) = @_;
2850         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, undef);
2851     });
2852
2853     foreach my $snapname (keys %{$conf->{snapshots}}) {
2854         my $snap = $conf->{snapshots}->{$snapname};
2855         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
2856         foreach_drive($snap, sub {
2857             my ($ds, $drive) = @_;
2858             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $snapname);
2859         });
2860     }
2861
2862     foreach my $volid (keys %$volhash) {
2863         &$func($volid, $volhash->{$volid}, @param);
2864     }
2865 }
2866
2867 sub conf_has_serial {
2868     my ($conf) = @_;
2869
2870     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2871         if ($conf->{"serial$i"}) {
2872             return 1;
2873         }
2874     }
2875
2876     return 0;
2877 }
2878
2879 sub vga_conf_has_spice {
2880     my ($vga) = @_;
2881
2882     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2883
2884     return $1 || 1;
2885 }
2886
2887 sub config_to_command {
2888     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2889
2890     my $cmd = [];
2891     my $globalFlags = [];
2892     my $machineFlags = [];
2893     my $rtcFlags = [];
2894     my $cpuFlags = [];
2895     my $devices = [];
2896     my $pciaddr = '';
2897     my $bridges = {};
2898     my $kvmver = kvm_user_version();
2899     my $vernum = 0; # unknown
2900     my $ostype = $conf->{ostype};
2901     my $winversion = windows_version($ostype);
2902     my $kvm = $conf->{kvm} // 1;
2903
2904     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
2905
2906     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2907         $vernum = $1*1000000+$2*1000;
2908     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2909         $vernum = $1*1000000+$2*1000+$3;
2910     }
2911
2912     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2913
2914     my $have_ovz = -f '/proc/vz/vestat';
2915
2916     my $q35 = machine_type_is_q35($conf);
2917     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2918     my $machine_type = $forcemachine || $conf->{machine};
2919     my $use_old_bios_files = undef;
2920     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2921
2922     my $cpuunits = defined($conf->{cpuunits}) ?
2923             $conf->{cpuunits} : $defaults->{cpuunits};
2924
2925     push @$cmd, '/usr/bin/kvm';
2926
2927     push @$cmd, '-id', $vmid;
2928
2929     my $use_virtio = 0;
2930
2931     my $qmpsocket = qmp_socket($vmid);
2932     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2933     push @$cmd, '-mon', "chardev=qmp,mode=control";
2934
2935
2936     push @$cmd, '-pidfile' , pidfile_name($vmid);
2937
2938     push @$cmd, '-daemonize';
2939
2940     if ($conf->{smbios1}) {
2941         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2942     }
2943
2944     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2945         die "uefi base image not found\n" if ! -f $OVMF_CODE;
2946
2947         my $path;
2948         my $format;
2949         if (my $efidisk = $conf->{efidisk0}) {
2950             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
2951             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2952             $format = $d->{format};
2953             if ($storeid) {
2954                 $path = PVE::Storage::path($storecfg, $d->{file});
2955                 if (!defined($format)) {
2956                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2957                     $format = qemu_img_format($scfg, $volname);
2958                 }
2959             } else {
2960                 $path = $d->{file};
2961                 die "efidisk format must be specified\n"
2962                     if !defined($format);
2963             }
2964         } else {
2965             warn "no efidisk configured! Using temporary efivars disk.\n";
2966             $path = "/tmp/$vmid-ovmf.fd";
2967             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
2968             $format = 'raw';
2969         }
2970
2971         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
2972         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
2973     }
2974
2975
2976     # add usb controllers
2977     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2978     push @$devices, @usbcontrollers if @usbcontrollers;
2979     my $vga = $conf->{vga};
2980
2981     my $qxlnum = vga_conf_has_spice($vga);
2982     $vga = 'qxl' if $qxlnum;
2983
2984     if (!$vga) {
2985         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
2986             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
2987         } else {
2988             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
2989         }
2990     }
2991
2992     # enable absolute mouse coordinates (needed by vnc)
2993     my $tablet;
2994     if (defined($conf->{tablet})) {
2995         $tablet = $conf->{tablet};
2996     } else {
2997         $tablet = $defaults->{tablet};
2998         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
2999         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3000     }
3001
3002     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3003
3004     my $kvm_off = 0;
3005     my $gpu_passthrough;
3006
3007     # host pci devices
3008     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3009         my $d = parse_hostpci($conf->{"hostpci$i"});
3010         next if !$d;
3011
3012         my $pcie = $d->{pcie};
3013         if($pcie){
3014             die "q35 machine model is not enabled" if !$q35;
3015             $pciaddr = print_pcie_addr("hostpci$i");
3016         }else{
3017             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3018         }
3019
3020         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3021         my $romfile = $d->{romfile};
3022
3023         my $xvga = '';
3024         if ($d->{'x-vga'}) {
3025             $xvga = ',x-vga=on';
3026             $kvm_off = 1;
3027             $vga = 'none';
3028             $gpu_passthrough = 1;
3029
3030             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3031                 $xvga = "";
3032             }
3033         }
3034         my $pcidevices = $d->{pciid};
3035         my $multifunction = 1 if @$pcidevices > 1;
3036
3037         my $j=0;
3038         foreach my $pcidevice (@$pcidevices) {
3039
3040             my $id = "hostpci$i";
3041             $id .= ".$j" if $multifunction;
3042             my $addr = $pciaddr;
3043             $addr .= ".$j" if $multifunction;
3044             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3045
3046             if($j == 0){
3047                 $devicestr .= "$rombar$xvga";
3048                 $devicestr .= ",multifunction=on" if $multifunction;
3049                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3050             }
3051
3052             push @$devices, '-device', $devicestr;
3053             $j++;
3054         }
3055     }
3056
3057     # usb devices
3058     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3059     push @$devices, @usbdevices if @usbdevices;
3060     # serial devices
3061     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3062         if (my $path = $conf->{"serial$i"}) {
3063             if ($path eq 'socket') {
3064                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3065                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3066                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3067             } else {
3068                 die "no such serial device\n" if ! -c $path;
3069                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3070                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3071             }
3072         }
3073     }
3074
3075     # parallel devices
3076     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3077         if (my $path = $conf->{"parallel$i"}) {
3078             die "no such parallel device\n" if ! -c $path;
3079             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3080             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3081             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3082         }
3083     }
3084
3085     my $vmname = $conf->{name} || "vm$vmid";
3086
3087     push @$cmd, '-name', $vmname;
3088
3089     my $sockets = 1;
3090     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3091     $sockets = $conf->{sockets} if  $conf->{sockets};
3092
3093     my $cores = $conf->{cores} || 1;
3094
3095     my $maxcpus = $sockets * $cores;
3096
3097     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3098
3099     my $allowed_vcpus = $cpuinfo->{cpus};
3100
3101     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3102         if ($allowed_vcpus < $maxcpus);
3103
3104     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3105
3106         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3107         for (my $i = 2; $i <= $vcpus; $i++)  {
3108             my $cpustr = print_cpu_device($conf,$i);
3109             push @$cmd, '-device', $cpustr;
3110         }
3111
3112     } else {
3113
3114         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3115     }
3116     push @$cmd, '-nodefaults';
3117
3118     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3119
3120     my $bootindex_hash = {};
3121     my $i = 1;
3122     foreach my $o (split(//, $bootorder)) {
3123         $bootindex_hash->{$o} = $i*100;
3124         $i++;
3125     }
3126
3127     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3128
3129     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3130
3131     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3132
3133     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3134
3135     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3136         my $socket = vnc_socket($vmid);
3137         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3138     } else {
3139         push @$cmd, '-nographic';
3140     }
3141
3142     # time drift fix
3143     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3144
3145     my $useLocaltime = $conf->{localtime};
3146
3147     if ($winversion >= 5) { # windows
3148         $useLocaltime = 1 if !defined($conf->{localtime});
3149
3150         # use time drift fix when acpi is enabled
3151         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3152             $tdf = 1 if !defined($conf->{tdf});
3153         }
3154     }
3155
3156     if ($winversion >= 6) {
3157         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3158         push @$cmd, '-no-hpet';
3159     }
3160
3161     push @$rtcFlags, 'driftfix=slew' if $tdf;
3162
3163     if (!$kvm) {
3164         push @$machineFlags, 'accel=tcg';
3165     }
3166
3167     if ($machine_type) {
3168         push @$machineFlags, "type=${machine_type}";
3169     }
3170
3171     if ($conf->{startdate}) {
3172         push @$rtcFlags, "base=$conf->{startdate}";
3173     } elsif ($useLocaltime) {
3174         push @$rtcFlags, 'base=localtime';
3175     }
3176
3177     my $cpu = $kvm ? "kvm64" : "qemu64";
3178     if (my $cputype = $conf->{cpu}) {
3179         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3180             or die "Cannot parse cpu description: $cputype\n";
3181         $cpu = $cpuconf->{cputype};
3182         $kvm_off = 1 if $cpuconf->{hidden};
3183
3184         if (defined(my $flags = $cpuconf->{flags})) {
3185             push @$cpuFlags, split(";", $flags);
3186         }
3187     }
3188
3189     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3190
3191     push @$cpuFlags , '-x2apic'
3192         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3193
3194     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3195
3196     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3197
3198     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3199
3200         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3201         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3202     }
3203
3204     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3205
3206     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3207
3208     push @$cpuFlags, 'kvm=off' if $kvm_off;
3209
3210     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3211         die "internal error"; # should not happen
3212
3213     push @$cpuFlags, "vendor=${cpu_vendor}"
3214         if $cpu_vendor ne 'default';
3215
3216     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3217
3218     push @$cmd, '-cpu', $cpu;
3219
3220     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3221
3222     push @$cmd, '-S' if $conf->{freeze};
3223
3224     # set keyboard layout
3225     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3226     push @$cmd, '-k', $kb if $kb;
3227
3228     # enable sound
3229     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3230     #push @$cmd, '-soundhw', 'es1370';
3231     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3232
3233     if($conf->{agent}) {
3234         my $qgasocket = qmp_socket($vmid, 1);
3235         my $pciaddr = print_pci_addr("qga0", $bridges);
3236         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3237         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3238         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3239     }
3240
3241     my $spice_port;
3242
3243     if ($qxlnum) {
3244         if ($qxlnum > 1) {
3245             if ($winversion){
3246                 for(my $i = 1; $i < $qxlnum; $i++){
3247                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3248                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3249                 }
3250             } else {
3251                 # assume other OS works like Linux
3252                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3253                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3254             }
3255         }
3256
3257         my $pciaddr = print_pci_addr("spice", $bridges);
3258
3259         my $nodename = PVE::INotify::nodename();
3260         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3261         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3262         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3263         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3264         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3265
3266         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3267
3268         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3269         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3270         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3271     }
3272
3273     # enable balloon by default, unless explicitly disabled
3274     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3275         $pciaddr = print_pci_addr("balloon0", $bridges);
3276         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3277     }
3278
3279     if ($conf->{watchdog}) {
3280         my $wdopts = parse_watchdog($conf->{watchdog});
3281         $pciaddr = print_pci_addr("watchdog", $bridges);
3282         my $watchdog = $wdopts->{model} || 'i6300esb';
3283         push @$devices, '-device', "$watchdog$pciaddr";
3284         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3285     }
3286
3287     my $vollist = [];
3288     my $scsicontroller = {};
3289     my $ahcicontroller = {};
3290     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3291
3292     # Add iscsi initiator name if available
3293     if (my $initiator = get_initiator_name()) {
3294         push @$devices, '-iscsi', "initiator-name=$initiator";
3295     }
3296
3297     foreach_drive($conf, sub {
3298         my ($ds, $drive) = @_;
3299
3300         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3301             push @$vollist, $drive->{file};
3302         }
3303
3304         # ignore efidisk here, already added in bios/fw handling code above
3305         return if $drive->{interface} eq 'efidisk';
3306
3307         $use_virtio = 1 if $ds =~ m/^virtio/;
3308
3309         if (drive_is_cdrom ($drive)) {
3310             if ($bootindex_hash->{d}) {
3311                 $drive->{bootindex} = $bootindex_hash->{d};
3312                 $bootindex_hash->{d} += 1;
3313             }
3314         } else {
3315             if ($bootindex_hash->{c}) {
3316                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3317                 $bootindex_hash->{c} += 1;
3318             }
3319         }
3320
3321         if($drive->{interface} eq 'virtio'){
3322            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3323         }
3324
3325         if ($drive->{interface} eq 'scsi') {
3326
3327             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3328
3329             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3330             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3331
3332             my $iothread = '';
3333             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3334                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3335                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3336             } elsif ($drive->{iothread}) {
3337                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3338             }
3339
3340             my $queues = '';
3341             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3342                 $queues = ",num_queues=$drive->{queues}";
3343             }
3344
3345             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3346             $scsicontroller->{$controller}=1;
3347         }
3348
3349         if ($drive->{interface} eq 'sata') {
3350            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3351            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3352            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3353            $ahcicontroller->{$controller}=1;
3354         }
3355
3356         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3357         push @$devices, '-drive',$drive_cmd;
3358         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3359     });
3360
3361     for (my $i = 0; $i < $MAX_NETS; $i++) {
3362          next if !$conf->{"net$i"};
3363          my $d = parse_net($conf->{"net$i"});
3364          next if !$d;
3365
3366          $use_virtio = 1 if $d->{model} eq 'virtio';
3367
3368          if ($bootindex_hash->{n}) {
3369             $d->{bootindex} = $bootindex_hash->{n};
3370             $bootindex_hash->{n} += 1;
3371          }
3372
3373          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3374          push @$devices, '-netdev', $netdevfull;
3375
3376          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3377          push @$devices, '-device', $netdevicefull;
3378     }
3379
3380     if (!$q35) {
3381         # add pci bridges
3382         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3383            $bridges->{1} = 1;
3384            $bridges->{2} = 1;
3385         }
3386
3387         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3388
3389         while (my ($k, $v) = each %$bridges) {
3390             $pciaddr = print_pci_addr("pci.$k");
3391             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3392         }
3393     }
3394
3395     # add custom args
3396     if ($conf->{args}) {
3397         my $aa = PVE::Tools::split_args($conf->{args});
3398         push @$cmd, @$aa;
3399     }
3400
3401     push @$cmd, @$devices;
3402     push @$cmd, '-rtc', join(',', @$rtcFlags)
3403         if scalar(@$rtcFlags);
3404     push @$cmd, '-machine', join(',', @$machineFlags)
3405         if scalar(@$machineFlags);
3406     push @$cmd, '-global', join(',', @$globalFlags)
3407         if scalar(@$globalFlags);
3408
3409     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3410 }
3411
3412 sub vnc_socket {
3413     my ($vmid) = @_;
3414     return "${var_run_tmpdir}/$vmid.vnc";
3415 }
3416
3417 sub spice_port {
3418     my ($vmid) = @_;
3419
3420     my $res = vm_mon_cmd($vmid, 'query-spice');
3421
3422     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3423 }
3424
3425 sub qmp_socket {
3426     my ($vmid, $qga) = @_;
3427     my $sockettype = $qga ? 'qga' : 'qmp';
3428     return "${var_run_tmpdir}/$vmid.$sockettype";
3429 }
3430
3431 sub pidfile_name {
3432     my ($vmid) = @_;
3433     return "${var_run_tmpdir}/$vmid.pid";
3434 }
3435
3436 sub vm_devices_list {
3437     my ($vmid) = @_;
3438
3439     my $res = vm_mon_cmd($vmid, 'query-pci');
3440     my $devices = {};
3441     foreach my $pcibus (@$res) {
3442         foreach my $device (@{$pcibus->{devices}}) {
3443             next if !$device->{'qdev_id'};
3444             if ($device->{'pci_bridge'}) {
3445                 $devices->{$device->{'qdev_id'}} = 1;
3446                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3447                     next if !$bridge_device->{'qdev_id'};
3448                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3449                     $devices->{$device->{'qdev_id'}}++;
3450                 }
3451             } else {
3452                 $devices->{$device->{'qdev_id'}} = 1;
3453             }
3454         }
3455     }
3456
3457     my $resblock = vm_mon_cmd($vmid, 'query-block');
3458     foreach my $block (@$resblock) {
3459         if($block->{device} =~ m/^drive-(\S+)/){
3460                 $devices->{$1} = 1;
3461         }
3462     }
3463
3464     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3465     foreach my $mice (@$resmice) {
3466         if ($mice->{name} eq 'QEMU HID Tablet') {
3467             $devices->{tablet} = 1;
3468             last;
3469         }
3470     }
3471
3472     # for usb devices there is no query-usb
3473     # but we can iterate over the entries in
3474     # qom-list path=/machine/peripheral
3475     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3476     foreach my $per (@$resperipheral) {
3477         if ($per->{name} =~ m/^usb\d+$/) {
3478             $devices->{$per->{name}} = 1;
3479         }
3480     }
3481
3482     return $devices;
3483 }
3484
3485 sub vm_deviceplug {
3486     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3487
3488     my $q35 = machine_type_is_q35($conf);
3489
3490     my $devices_list = vm_devices_list($vmid);
3491     return 1 if defined($devices_list->{$deviceid});
3492
3493     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3494
3495     if ($deviceid eq 'tablet') {
3496
3497         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3498
3499     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3500
3501         die "usb hotplug currently not reliable\n";
3502         # since we can't reliably hot unplug all added usb devices
3503         # and usb passthrough disables live migration
3504         # we disable usb hotplugging for now
3505         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3506
3507     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3508
3509         qemu_iothread_add($vmid, $deviceid, $device);
3510
3511         qemu_driveadd($storecfg, $vmid, $device);
3512         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3513
3514         qemu_deviceadd($vmid, $devicefull);
3515         eval { qemu_deviceaddverify($vmid, $deviceid); };
3516         if (my $err = $@) {
3517             eval { qemu_drivedel($vmid, $deviceid); };
3518             warn $@ if $@;
3519             die $err;
3520         }
3521
3522     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3523
3524
3525         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3526         my $pciaddr = print_pci_addr($deviceid);
3527         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3528
3529         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3530
3531         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3532             qemu_iothread_add($vmid, $deviceid, $device);
3533             $devicefull .= ",iothread=iothread-$deviceid";
3534         }
3535
3536         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3537             $devicefull .= ",num_queues=$device->{queues}";
3538         }
3539
3540         qemu_deviceadd($vmid, $devicefull);
3541         qemu_deviceaddverify($vmid, $deviceid);
3542
3543     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3544
3545         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3546         qemu_driveadd($storecfg, $vmid, $device);
3547
3548         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3549         eval { qemu_deviceadd($vmid, $devicefull); };
3550         if (my $err = $@) {
3551             eval { qemu_drivedel($vmid, $deviceid); };
3552             warn $@ if $@;
3553             die $err;
3554         }
3555
3556     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3557
3558         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3559
3560         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3561         my $use_old_bios_files = undef;
3562         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3563
3564         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3565         qemu_deviceadd($vmid, $netdevicefull);
3566         eval { qemu_deviceaddverify($vmid, $deviceid); };
3567         if (my $err = $@) {
3568             eval { qemu_netdevdel($vmid, $deviceid); };
3569             warn $@ if $@;
3570             die $err;
3571         }
3572
3573     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3574
3575         my $bridgeid = $2;
3576         my $pciaddr = print_pci_addr($deviceid);
3577         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3578
3579         qemu_deviceadd($vmid, $devicefull);
3580         qemu_deviceaddverify($vmid, $deviceid);
3581
3582     } else {
3583         die "can't hotplug device '$deviceid'\n";
3584     }
3585
3586     return 1;
3587 }
3588
3589 # fixme: this should raise exceptions on error!
3590 sub vm_deviceunplug {
3591     my ($vmid, $conf, $deviceid) = @_;
3592
3593     my $devices_list = vm_devices_list($vmid);
3594     return 1 if !defined($devices_list->{$deviceid});
3595
3596     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3597
3598     if ($deviceid eq 'tablet') {
3599
3600         qemu_devicedel($vmid, $deviceid);
3601
3602     } elsif ($deviceid =~ m/^usb\d+$/) {
3603
3604         die "usb hotplug currently not reliable\n";
3605         # when unplugging usb devices this way,
3606         # there may be remaining usb controllers/hubs
3607         # so we disable it for now
3608         qemu_devicedel($vmid, $deviceid);
3609         qemu_devicedelverify($vmid, $deviceid);
3610
3611     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3612
3613         qemu_devicedel($vmid, $deviceid);
3614         qemu_devicedelverify($vmid, $deviceid);
3615         qemu_drivedel($vmid, $deviceid);
3616         qemu_iothread_del($conf, $vmid, $deviceid);
3617
3618     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3619
3620         qemu_devicedel($vmid, $deviceid);
3621         qemu_devicedelverify($vmid, $deviceid);
3622         qemu_iothread_del($conf, $vmid, $deviceid);
3623
3624     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3625
3626         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3627         my $device = parse_drive($deviceid, $conf->{$deviceid});
3628         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3629
3630         qemu_devicedel($vmid, $deviceid);
3631         qemu_drivedel($vmid, $deviceid);
3632         qemu_deletescsihw($conf, $vmid, $deviceid);
3633
3634     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3635
3636         qemu_devicedel($vmid, $deviceid);
3637         qemu_devicedelverify($vmid, $deviceid);
3638         qemu_netdevdel($vmid, $deviceid);
3639
3640     } else {
3641         die "can't unplug device '$deviceid'\n";
3642     }
3643
3644     return 1;
3645 }
3646
3647 sub qemu_deviceadd {
3648     my ($vmid, $devicefull) = @_;
3649
3650     $devicefull = "driver=".$devicefull;
3651     my %options =  split(/[=,]/, $devicefull);
3652
3653     vm_mon_cmd($vmid, "device_add" , %options);
3654 }
3655
3656 sub qemu_devicedel {
3657     my ($vmid, $deviceid) = @_;
3658
3659     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3660 }
3661
3662 sub qemu_iothread_add {
3663     my($vmid, $deviceid, $device) = @_;
3664
3665     if ($device->{iothread}) {
3666         my $iothreads = vm_iothreads_list($vmid);
3667         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3668     }
3669 }
3670
3671 sub qemu_iothread_del {
3672     my($conf, $vmid, $deviceid) = @_;
3673
3674     my $device = parse_drive($deviceid, $conf->{$deviceid});
3675     if ($device->{iothread}) {
3676         my $iothreads = vm_iothreads_list($vmid);
3677         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3678     }
3679 }
3680
3681 sub qemu_objectadd {
3682     my($vmid, $objectid, $qomtype) = @_;
3683
3684     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3685
3686     return 1;
3687 }
3688
3689 sub qemu_objectdel {
3690     my($vmid, $objectid) = @_;
3691
3692     vm_mon_cmd($vmid, "object-del", id => $objectid);
3693
3694     return 1;
3695 }
3696
3697 sub qemu_driveadd {
3698     my ($storecfg, $vmid, $device) = @_;
3699
3700     my $drive = print_drive_full($storecfg, $vmid, $device);
3701     $drive =~ s/\\/\\\\/g;
3702     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3703
3704     # If the command succeeds qemu prints: "OK"
3705     return 1 if $ret =~ m/OK/s;
3706
3707     die "adding drive failed: $ret\n";
3708 }
3709
3710 sub qemu_drivedel {
3711     my($vmid, $deviceid) = @_;
3712
3713     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3714     $ret =~ s/^\s+//;
3715
3716     return 1 if $ret eq "";
3717
3718     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3719     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3720
3721     die "deleting drive $deviceid failed : $ret\n";
3722 }
3723
3724 sub qemu_deviceaddverify {
3725     my ($vmid, $deviceid) = @_;
3726
3727     for (my $i = 0; $i <= 5; $i++) {
3728          my $devices_list = vm_devices_list($vmid);
3729          return 1 if defined($devices_list->{$deviceid});
3730          sleep 1;
3731     }
3732
3733     die "error on hotplug device '$deviceid'\n";
3734 }
3735
3736
3737 sub qemu_devicedelverify {
3738     my ($vmid, $deviceid) = @_;
3739
3740     # need to verify that the device is correctly removed as device_del
3741     # is async and empty return is not reliable
3742
3743     for (my $i = 0; $i <= 5; $i++) {
3744          my $devices_list = vm_devices_list($vmid);
3745          return 1 if !defined($devices_list->{$deviceid});
3746          sleep 1;
3747     }
3748
3749     die "error on hot-unplugging device '$deviceid'\n";
3750 }
3751
3752 sub qemu_findorcreatescsihw {
3753     my ($storecfg, $conf, $vmid, $device) = @_;
3754
3755     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3756
3757     my $scsihwid="$controller_prefix$controller";
3758     my $devices_list = vm_devices_list($vmid);
3759
3760     if(!defined($devices_list->{$scsihwid})) {
3761         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3762     }
3763
3764     return 1;
3765 }
3766
3767 sub qemu_deletescsihw {
3768     my ($conf, $vmid, $opt) = @_;
3769
3770     my $device = parse_drive($opt, $conf->{$opt});
3771
3772     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3773         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3774         return 1;
3775     }
3776
3777     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3778
3779     my $devices_list = vm_devices_list($vmid);
3780     foreach my $opt (keys %{$devices_list}) {
3781         if (PVE::QemuServer::is_valid_drivename($opt)) {
3782             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3783             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3784                 return 1;
3785             }
3786         }
3787     }
3788
3789     my $scsihwid="scsihw$controller";
3790
3791     vm_deviceunplug($vmid, $conf, $scsihwid);
3792
3793     return 1;
3794 }
3795
3796 sub qemu_add_pci_bridge {
3797     my ($storecfg, $conf, $vmid, $device) = @_;
3798
3799     my $bridges = {};
3800
3801     my $bridgeid;
3802
3803     print_pci_addr($device, $bridges);
3804
3805     while (my ($k, $v) = each %$bridges) {
3806         $bridgeid = $k;
3807     }
3808     return 1 if !defined($bridgeid) || $bridgeid < 1;
3809
3810     my $bridge = "pci.$bridgeid";
3811     my $devices_list = vm_devices_list($vmid);
3812
3813     if (!defined($devices_list->{$bridge})) {
3814         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3815     }
3816
3817     return 1;
3818 }
3819
3820 sub qemu_set_link_status {
3821     my ($vmid, $device, $up) = @_;
3822
3823     vm_mon_cmd($vmid, "set_link", name => $device,
3824                up => $up ? JSON::true : JSON::false);
3825 }
3826
3827 sub qemu_netdevadd {
3828     my ($vmid, $conf, $device, $deviceid) = @_;
3829
3830     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3831     my %options =  split(/[=,]/, $netdev);
3832
3833     vm_mon_cmd($vmid, "netdev_add",  %options);
3834     return 1;
3835 }
3836
3837 sub qemu_netdevdel {
3838     my ($vmid, $deviceid) = @_;
3839
3840     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3841 }
3842
3843 sub qemu_usb_hotplug {
3844     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3845
3846     return if !$device;
3847
3848     # remove the old one first
3849     vm_deviceunplug($vmid, $conf, $deviceid);
3850
3851     # check if xhci controller is necessary and available
3852     if ($device->{usb3}) {
3853
3854         my $devicelist = vm_devices_list($vmid);
3855
3856         if (!$devicelist->{xhci}) {
3857             my $pciaddr = print_pci_addr("xhci");
3858             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3859         }
3860     }
3861     my $d = parse_usb_device($device->{host});
3862     $d->{usb3} = $device->{usb3};
3863
3864     # add the new one
3865     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3866 }
3867
3868 sub qemu_cpu_hotplug {
3869     my ($vmid, $conf, $vcpus) = @_;
3870
3871     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3872
3873     my $sockets = 1;
3874     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3875     $sockets = $conf->{sockets} if  $conf->{sockets};
3876     my $cores = $conf->{cores} || 1;
3877     my $maxcpus = $sockets * $cores;
3878
3879     $vcpus = $maxcpus if !$vcpus;
3880
3881     die "you can't add more vcpus than maxcpus\n"
3882         if $vcpus > $maxcpus;
3883
3884     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3885
3886     if ($vcpus < $currentvcpus) {
3887
3888         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3889
3890             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3891                 qemu_devicedel($vmid, "cpu$i");
3892                 my $retry = 0;
3893                 my $currentrunningvcpus = undef;
3894                 while (1) {
3895                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3896                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3897                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3898                     $retry++;
3899                     sleep 1;
3900                 }
3901                 #update conf after each succesfull cpu unplug
3902                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3903                 PVE::QemuConfig->write_config($vmid, $conf);
3904             }
3905         } else {
3906             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3907         }
3908
3909         return;
3910     }
3911
3912     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3913     die "vcpus in running vm does not match its configuration\n"
3914         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3915
3916     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3917
3918         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3919             my $cpustr = print_cpu_device($conf, $i);
3920             qemu_deviceadd($vmid, $cpustr);
3921
3922             my $retry = 0;
3923             my $currentrunningvcpus = undef;
3924             while (1) {
3925                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3926                 last if scalar(@{$currentrunningvcpus}) == $i;
3927                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3928                 sleep 1;
3929                 $retry++;
3930             }
3931             #update conf after each succesfull cpu hotplug
3932             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3933             PVE::QemuConfig->write_config($vmid, $conf);
3934         }
3935     } else {
3936
3937         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3938             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3939         }
3940     }
3941 }
3942
3943 sub qemu_block_set_io_throttle {
3944     my ($vmid, $deviceid,
3945         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3946         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3947         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3948         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3949
3950     return if !check_running($vmid) ;
3951
3952     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3953         bps => int($bps),
3954         bps_rd => int($bps_rd),
3955         bps_wr => int($bps_wr),
3956         iops => int($iops),
3957         iops_rd => int($iops_rd),
3958         iops_wr => int($iops_wr),
3959         bps_max => int($bps_max),
3960         bps_rd_max => int($bps_rd_max),
3961         bps_wr_max => int($bps_wr_max),
3962         iops_max => int($iops_max),
3963         iops_rd_max => int($iops_rd_max),
3964         iops_wr_max => int($iops_wr_max),
3965         bps_max_length => int($bps_max_length),
3966         bps_rd_max_length => int($bps_rd_max_length),
3967         bps_wr_max_length => int($bps_wr_max_length),
3968         iops_max_length => int($iops_max_length),
3969         iops_rd_max_length => int($iops_rd_max_length),
3970         iops_wr_max_length => int($iops_wr_max_length),
3971     );
3972
3973 }
3974
3975 # old code, only used to shutdown old VM after update
3976 sub __read_avail {
3977     my ($fh, $timeout) = @_;
3978
3979     my $sel = new IO::Select;
3980     $sel->add($fh);
3981
3982     my $res = '';
3983     my $buf;
3984
3985     my @ready;
3986     while (scalar (@ready = $sel->can_read($timeout))) {
3987         my $count;
3988         if ($count = $fh->sysread($buf, 8192)) {
3989             if ($buf =~ /^(.*)\(qemu\) $/s) {
3990                 $res .= $1;
3991                 last;
3992             } else {
3993                 $res .= $buf;
3994             }
3995         } else {
3996             if (!defined($count)) {
3997                 die "$!\n";
3998             }
3999             last;
4000         }
4001     }
4002
4003     die "monitor read timeout\n" if !scalar(@ready);
4004
4005     return $res;
4006 }
4007
4008 # old code, only used to shutdown old VM after update
4009 sub vm_monitor_command {
4010     my ($vmid, $cmdstr, $nocheck) = @_;
4011
4012     my $res;
4013
4014     eval {
4015         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4016
4017         my $sname = "${var_run_tmpdir}/$vmid.mon";
4018
4019         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
4020             die "unable to connect to VM $vmid socket - $!\n";
4021
4022         my $timeout = 3;
4023
4024         # hack: migrate sometime blocks the monitor (when migrate_downtime
4025         # is set)
4026         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4027             $timeout = 60*60; # 1 hour
4028         }
4029
4030         # read banner;
4031         my $data = __read_avail($sock, $timeout);
4032
4033         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
4034             die "got unexpected qemu monitor banner\n";
4035         }
4036
4037         my $sel = new IO::Select;
4038         $sel->add($sock);
4039
4040         if (!scalar(my @ready = $sel->can_write($timeout))) {
4041             die "monitor write error - timeout";
4042         }
4043
4044         my $fullcmd = "$cmdstr\r";
4045
4046         # syslog('info', "VM $vmid monitor command: $cmdstr");
4047
4048         my $b;
4049         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
4050             die "monitor write error - $!";
4051         }
4052
4053         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4054
4055         $timeout = 20;
4056
4057         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4058             $timeout = 60*60; # 1 hour
4059         } elsif ($cmdstr =~ m/^(eject|change)/) {
4060             $timeout = 60; # note: cdrom mount command is slow
4061         }
4062         if ($res = __read_avail($sock, $timeout)) {
4063
4064             my @lines = split("\r?\n", $res);
4065
4066             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4067
4068             $res = join("\n", @lines);
4069             $res .= "\n";
4070         }
4071     };
4072
4073     my $err = $@;
4074
4075     if ($err) {
4076         syslog("err", "VM $vmid monitor command failed - $err");
4077         die $err;
4078     }
4079
4080     return $res;
4081 }
4082
4083 sub qemu_block_resize {
4084     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4085
4086     my $running = check_running($vmid);
4087
4088     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4089
4090     return if !$running;
4091
4092     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4093
4094 }
4095
4096 sub qemu_volume_snapshot {
4097     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4098
4099     my $running = check_running($vmid);
4100
4101     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4102         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4103     } else {
4104         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4105     }
4106 }
4107
4108 sub qemu_volume_snapshot_delete {
4109     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4110
4111     my $running = check_running($vmid);
4112
4113     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4114         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4115     } else {
4116         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4117     }
4118 }
4119
4120 sub set_migration_caps {
4121     my ($vmid) = @_;
4122
4123     my $cap_ref = [];
4124
4125     my $enabled_cap = {
4126         "auto-converge" => 1,
4127         "xbzrle" => 1,
4128         "x-rdma-pin-all" => 0,
4129         "zero-blocks" => 0,
4130         "compress" => 0
4131     };
4132
4133     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4134
4135     for my $supported_capability (@$supported_capabilities) {
4136         push @$cap_ref, {
4137             capability => $supported_capability->{capability},
4138             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4139         };
4140     }
4141
4142     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4143 }
4144
4145 my $fast_plug_option = {
4146     'lock' => 1,
4147     'name' => 1,
4148     'onboot' => 1,
4149     'shares' => 1,
4150     'startup' => 1,
4151     'description' => 1,
4152     'protection' => 1,
4153     'vmstatestorage' => 1,
4154 };
4155
4156 # hotplug changes in [PENDING]
4157 # $selection hash can be used to only apply specified options, for
4158 # example: { cores => 1 } (only apply changed 'cores')
4159 # $errors ref is used to return error messages
4160 sub vmconfig_hotplug_pending {
4161     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4162
4163     my $defaults = load_defaults();
4164
4165     # commit values which do not have any impact on running VM first
4166     # Note: those option cannot raise errors, we we do not care about
4167     # $selection and always apply them.
4168
4169     my $add_error = sub {
4170         my ($opt, $msg) = @_;
4171         $errors->{$opt} = "hotplug problem - $msg";
4172     };
4173
4174     my $changes = 0;
4175     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4176         if ($fast_plug_option->{$opt}) {
4177             $conf->{$opt} = $conf->{pending}->{$opt};
4178             delete $conf->{pending}->{$opt};
4179             $changes = 1;
4180         }
4181     }
4182
4183     if ($changes) {
4184         PVE::QemuConfig->write_config($vmid, $conf);
4185         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4186     }
4187
4188     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4189
4190     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4191     while (my ($opt, $force) = each %$pending_delete_hash) {
4192         next if $selection && !$selection->{$opt};
4193         eval {
4194             if ($opt eq 'hotplug') {
4195                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4196             } elsif ($opt eq 'tablet') {
4197                 die "skip\n" if !$hotplug_features->{usb};
4198                 if ($defaults->{tablet}) {
4199                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4200                 } else {
4201                     vm_deviceunplug($vmid, $conf, $opt);
4202                 }
4203             } elsif ($opt =~ m/^usb\d+/) {
4204                 die "skip\n";
4205                 # since we cannot reliably hot unplug usb devices
4206                 # we are disabling it
4207                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4208                 vm_deviceunplug($vmid, $conf, $opt);
4209             } elsif ($opt eq 'vcpus') {
4210                 die "skip\n" if !$hotplug_features->{cpu};
4211                 qemu_cpu_hotplug($vmid, $conf, undef);
4212             } elsif ($opt eq 'balloon') {
4213                 # enable balloon device is not hotpluggable
4214                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4215             } elsif ($fast_plug_option->{$opt}) {
4216                 # do nothing
4217             } elsif ($opt =~ m/^net(\d+)$/) {
4218                 die "skip\n" if !$hotplug_features->{network};
4219                 vm_deviceunplug($vmid, $conf, $opt);
4220             } elsif (is_valid_drivename($opt)) {
4221                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4222                 vm_deviceunplug($vmid, $conf, $opt);
4223                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4224             } elsif ($opt =~ m/^memory$/) {
4225                 die "skip\n" if !$hotplug_features->{memory};
4226                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4227             } elsif ($opt eq 'cpuunits') {
4228                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4229             } elsif ($opt eq 'cpulimit') {
4230                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4231             } else {
4232                 die "skip\n";
4233             }
4234         };
4235         if (my $err = $@) {
4236             &$add_error($opt, $err) if $err ne "skip\n";
4237         } else {
4238             # save new config if hotplug was successful
4239             delete $conf->{$opt};
4240             vmconfig_undelete_pending_option($conf, $opt);
4241             PVE::QemuConfig->write_config($vmid, $conf);
4242             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4243         }
4244     }
4245
4246     foreach my $opt (keys %{$conf->{pending}}) {
4247         next if $selection && !$selection->{$opt};
4248         my $value = $conf->{pending}->{$opt};
4249         eval {
4250             if ($opt eq 'hotplug') {
4251                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4252             } elsif ($opt eq 'tablet') {
4253                 die "skip\n" if !$hotplug_features->{usb};
4254                 if ($value == 1) {
4255                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4256                 } elsif ($value == 0) {
4257                     vm_deviceunplug($vmid, $conf, $opt);
4258                 }
4259             } elsif ($opt =~ m/^usb\d+$/) {
4260                 die "skip\n";
4261                 # since we cannot reliably hot unplug usb devices
4262                 # we are disabling it
4263                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4264                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4265                 die "skip\n" if !$d;
4266                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4267             } elsif ($opt eq 'vcpus') {
4268                 die "skip\n" if !$hotplug_features->{cpu};
4269                 qemu_cpu_hotplug($vmid, $conf, $value);
4270             } elsif ($opt eq 'balloon') {
4271                 # enable/disable balloning device is not hotpluggable
4272                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4273                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4274                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4275
4276                 # allow manual ballooning if shares is set to zero
4277                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4278                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4279                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4280                 }
4281             } elsif ($opt =~ m/^net(\d+)$/) {
4282                 # some changes can be done without hotplug
4283                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4284                                     $vmid, $opt, $value);
4285             } elsif (is_valid_drivename($opt)) {
4286                 # some changes can be done without hotplug
4287                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4288                                      $vmid, $opt, $value, 1);
4289             } elsif ($opt =~ m/^memory$/) { #dimms
4290                 die "skip\n" if !$hotplug_features->{memory};
4291                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4292             } elsif ($opt eq 'cpuunits') {
4293                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4294             } elsif ($opt eq 'cpulimit') {
4295                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4296                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4297             } else {
4298                 die "skip\n";  # skip non-hot-pluggable options
4299             }
4300         };
4301         if (my $err = $@) {
4302             &$add_error($opt, $err) if $err ne "skip\n";
4303         } else {
4304             # save new config if hotplug was successful
4305             $conf->{$opt} = $value;
4306             delete $conf->{pending}->{$opt};
4307             PVE::QemuConfig->write_config($vmid, $conf);
4308             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4309         }
4310     }
4311 }
4312
4313 sub try_deallocate_drive {
4314     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4315
4316     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4317         my $volid = $drive->{file};
4318         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4319             my $sid = PVE::Storage::parse_volume_id($volid);
4320             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4321
4322             # check if the disk is really unused
4323             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4324                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4325             PVE::Storage::vdisk_free($storecfg, $volid);
4326             return 1;
4327         } else {
4328             # If vm is not owner of this disk remove from config
4329             return 1;
4330         }
4331     }
4332
4333     return undef;
4334 }
4335
4336 sub vmconfig_delete_or_detach_drive {
4337     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4338
4339     my $drive = parse_drive($opt, $conf->{$opt});
4340
4341     my $rpcenv = PVE::RPCEnvironment::get();
4342     my $authuser = $rpcenv->get_user();
4343
4344     if ($force) {
4345         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4346         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4347     } else {
4348         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4349     }
4350 }
4351
4352 sub vmconfig_apply_pending {
4353     my ($vmid, $conf, $storecfg) = @_;
4354
4355     # cold plug
4356
4357     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4358     while (my ($opt, $force) = each %$pending_delete_hash) {
4359         die "internal error" if $opt =~ m/^unused/;
4360         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4361         if (!defined($conf->{$opt})) {
4362             vmconfig_undelete_pending_option($conf, $opt);
4363             PVE::QemuConfig->write_config($vmid, $conf);
4364         } elsif (is_valid_drivename($opt)) {
4365             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4366             vmconfig_undelete_pending_option($conf, $opt);
4367             delete $conf->{$opt};
4368             PVE::QemuConfig->write_config($vmid, $conf);
4369         } else {
4370             vmconfig_undelete_pending_option($conf, $opt);
4371             delete $conf->{$opt};
4372             PVE::QemuConfig->write_config($vmid, $conf);
4373         }
4374     }
4375
4376     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4377
4378     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4379         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4380
4381         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4382             # skip if nothing changed
4383         } elsif (is_valid_drivename($opt)) {
4384             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4385                 if defined($conf->{$opt});
4386             $conf->{$opt} = $conf->{pending}->{$opt};
4387         } else {
4388             $conf->{$opt} = $conf->{pending}->{$opt};
4389         }
4390
4391         delete $conf->{pending}->{$opt};
4392         PVE::QemuConfig->write_config($vmid, $conf);
4393     }
4394 }
4395
4396 my $safe_num_ne = sub {
4397     my ($a, $b) = @_;
4398
4399     return 0 if !defined($a) && !defined($b);
4400     return 1 if !defined($a);
4401     return 1 if !defined($b);
4402
4403     return $a != $b;
4404 };
4405
4406 my $safe_string_ne = sub {
4407     my ($a, $b) = @_;
4408
4409     return 0 if !defined($a) && !defined($b);
4410     return 1 if !defined($a);
4411     return 1 if !defined($b);
4412
4413     return $a ne $b;
4414 };
4415
4416 sub vmconfig_update_net {
4417     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4418
4419     my $newnet = parse_net($value);
4420
4421     if ($conf->{$opt}) {
4422         my $oldnet = parse_net($conf->{$opt});
4423
4424         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4425             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4426             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4427             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4428
4429             # for non online change, we try to hot-unplug
4430             die "skip\n" if !$hotplug;
4431             vm_deviceunplug($vmid, $conf, $opt);
4432         } else {
4433
4434             die "internal error" if $opt !~ m/net(\d+)/;
4435             my $iface = "tap${vmid}i$1";
4436
4437             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4438                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4439                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4440                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4441                 PVE::Network::tap_unplug($iface);
4442                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4443             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4444                 # Rate can be applied on its own but any change above needs to
4445                 # include the rate in tap_plug since OVS resets everything.
4446                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4447             }
4448
4449             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4450                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4451             }
4452
4453             return 1;
4454         }
4455     }
4456
4457     if ($hotplug) {
4458         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4459     } else {
4460         die "skip\n";
4461     }
4462 }
4463
4464 sub vmconfig_update_disk {
4465     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4466
4467     # fixme: do we need force?
4468
4469     my $drive = parse_drive($opt, $value);
4470
4471     if ($conf->{$opt}) {
4472
4473         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4474
4475             my $media = $drive->{media} || 'disk';
4476             my $oldmedia = $old_drive->{media} || 'disk';
4477             die "unable to change media type\n" if $media ne $oldmedia;
4478
4479             if (!drive_is_cdrom($old_drive)) {
4480
4481                 if ($drive->{file} ne $old_drive->{file}) {
4482
4483                     die "skip\n" if !$hotplug;
4484
4485                     # unplug and register as unused
4486                     vm_deviceunplug($vmid, $conf, $opt);
4487                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4488
4489                 } else {
4490                     # update existing disk
4491
4492                     # skip non hotpluggable value
4493                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4494                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4495                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4496                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4497                         die "skip\n";
4498                     }
4499
4500                     # apply throttle
4501                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4502                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4503                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4504                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4505                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4506                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4507                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4508                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4509                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4510                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4511                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4512                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4513                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4514                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4515                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4516                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4517                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4518                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4519
4520                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4521                                                    ($drive->{mbps} || 0)*1024*1024,
4522                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4523                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4524                                                    $drive->{iops} || 0,
4525                                                    $drive->{iops_rd} || 0,
4526                                                    $drive->{iops_wr} || 0,
4527                                                    ($drive->{mbps_max} || 0)*1024*1024,
4528                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4529                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4530                                                    $drive->{iops_max} || 0,
4531                                                    $drive->{iops_rd_max} || 0,
4532                                                    $drive->{iops_wr_max} || 0,
4533                                                    $drive->{bps_max_length} || 1,
4534                                                    $drive->{bps_rd_max_length} || 1,
4535                                                    $drive->{bps_wr_max_length} || 1,
4536                                                    $drive->{iops_max_length} || 1,
4537                                                    $drive->{iops_rd_max_length} || 1,
4538                                                    $drive->{iops_wr_max_length} || 1);
4539
4540                     }
4541
4542                     return 1;
4543                 }
4544
4545             } else { # cdrom
4546
4547                 if ($drive->{file} eq 'none') {
4548                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4549                 } else {
4550                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4551                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4552                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4553                 }
4554
4555                 return 1;
4556             }
4557         }
4558     }
4559
4560     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4561     # hotplug new disks
4562     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4563     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4564 }
4565
4566 sub vm_start {
4567     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4568         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4569
4570     PVE::QemuConfig->lock_config($vmid, sub {
4571         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4572
4573         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4574
4575         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4576
4577         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4578
4579         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4580             vmconfig_apply_pending($vmid, $conf, $storecfg);
4581             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4582         }
4583
4584         my $defaults = load_defaults();
4585
4586         # set environment variable useful inside network script
4587         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4588
4589         my $local_volumes = {};
4590
4591         if ($targetstorage) {
4592             foreach_drive($conf, sub {
4593                 my ($ds, $drive) = @_;
4594
4595                 return if drive_is_cdrom($drive);
4596
4597                 my $volid = $drive->{file};
4598
4599                 return if !$volid;
4600
4601                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4602
4603                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4604                 return if $scfg->{shared};
4605                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4606             });
4607
4608             my $format = undef;
4609
4610             foreach my $opt (sort keys %$local_volumes) {
4611
4612                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4613                 my $drive = parse_drive($opt, $conf->{$opt});
4614
4615                 #if remote storage is specified, use default format
4616                 if ($targetstorage && $targetstorage ne "1") {
4617                     $storeid = $targetstorage;
4618                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4619                     $format = $defFormat;
4620                 } else {
4621                     #else we use same format than original
4622                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4623                     $format = qemu_img_format($scfg, $volid);
4624                 }
4625
4626                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4627                 my $newdrive = $drive;
4628                 $newdrive->{format} = $format;
4629                 $newdrive->{file} = $newvolid;
4630                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4631                 $local_volumes->{$opt} = $drivestr;
4632                 #pass drive to conf for command line
4633                 $conf->{$opt} = $drivestr;
4634             }
4635         }
4636
4637         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4638
4639         my $migrate_port = 0;
4640         my $migrate_uri;
4641         if ($statefile) {
4642             if ($statefile eq 'tcp') {
4643                 my $localip = "localhost";
4644                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4645                 my $nodename = PVE::INotify::nodename();
4646
4647                 if (!defined($migration_type)) {
4648                     if (defined($datacenterconf->{migration}->{type})) {
4649                         $migration_type = $datacenterconf->{migration}->{type};
4650                     } else {
4651                         $migration_type = 'secure';
4652                     }
4653                 }
4654
4655                 if ($migration_type eq 'insecure') {
4656                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4657                     if ($migrate_network_addr) {
4658                         $localip = $migrate_network_addr;
4659                     } else {
4660                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4661                     }
4662
4663                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4664                 }
4665
4666                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4667                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4668                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4669                 push @$cmd, '-incoming', $migrate_uri;
4670                 push @$cmd, '-S';
4671
4672             } elsif ($statefile eq 'unix') {
4673                 # should be default for secure migrations as a ssh TCP forward
4674                 # tunnel is not deterministic reliable ready and fails regurarly
4675                 # to set up in time, so use UNIX socket forwards
4676                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4677                 unlink $socket_addr;
4678
4679                 $migrate_uri = "unix:$socket_addr";
4680
4681                 push @$cmd, '-incoming', $migrate_uri;
4682                 push @$cmd, '-S';
4683
4684             } else {
4685                 push @$cmd, '-loadstate', $statefile;
4686             }
4687         } elsif ($paused) {
4688             push @$cmd, '-S';
4689         }
4690
4691         # host pci devices
4692         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4693           my $d = parse_hostpci($conf->{"hostpci$i"});
4694           next if !$d;
4695           my $pcidevices = $d->{pciid};
4696           foreach my $pcidevice (@$pcidevices) {
4697                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4698
4699                 my $info = pci_device_info("0000:$pciid");
4700                 die "IOMMU not present\n" if !check_iommu_support();
4701                 die "no pci device info for device '$pciid'\n" if !$info;
4702                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4703                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4704           }
4705         }
4706
4707         PVE::Storage::activate_volumes($storecfg, $vollist);
4708
4709         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4710             my $cmd = [];
4711             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4712             eval  { run_command($cmd); };
4713         }
4714
4715         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4716                                                   : $defaults->{cpuunits};
4717
4718         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4719         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4720
4721         my %properties = (
4722             Slice => 'qemu.slice',
4723             KillMode => 'none',
4724             CPUShares => $cpuunits
4725         );
4726
4727         if (my $cpulimit = $conf->{cpulimit}) {
4728             $properties{CPUQuota} = int($cpulimit * 100);
4729         }
4730         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4731
4732         if ($conf->{hugepages}) {
4733
4734             my $code = sub {
4735                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4736                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4737
4738                 PVE::QemuServer::Memory::hugepages_mount();
4739                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4740
4741                 eval  {
4742                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4743                     run_command($cmd, %run_params);
4744                 };
4745
4746                 if (my $err = $@) {
4747                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4748                     die $err;
4749                 }
4750
4751                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4752             };
4753             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4754
4755         } else {
4756             eval  {
4757                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4758                 run_command($cmd, %run_params);
4759             };
4760         }
4761
4762         if (my $err = $@) {
4763             # deactivate volumes if start fails
4764             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4765             die "start failed: $err";
4766         }
4767
4768         print "migration listens on $migrate_uri\n" if $migrate_uri;
4769
4770         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4771             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4772             warn $@ if $@;
4773         }
4774
4775         #start nbd server for storage migration
4776         if ($targetstorage) {
4777             my $nodename = PVE::INotify::nodename();
4778             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4779             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4780             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4781             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4782
4783             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4784
4785             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4786
4787             foreach my $opt (sort keys %$local_volumes) {
4788                 my $volid = $local_volumes->{$opt};
4789                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4790                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4791                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4792             }
4793         }
4794
4795         if ($migratedfrom) {
4796             eval {
4797                 set_migration_caps($vmid);
4798             };
4799             warn $@ if $@;
4800
4801             if ($spice_port) {
4802                 print "spice listens on port $spice_port\n";
4803                 if ($spice_ticket) {
4804                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4805                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4806                 }
4807             }
4808
4809         } else {
4810             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4811                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4812                     if $conf->{balloon};
4813             }
4814
4815             foreach my $opt (keys %$conf) {
4816                 next if $opt !~  m/^net\d+$/;
4817                 my $nicconf = parse_net($conf->{$opt});
4818                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4819             }
4820         }
4821
4822         vm_mon_cmd_nocheck($vmid, 'qom-set',
4823                     path => "machine/peripheral/balloon0",
4824                     property => "guest-stats-polling-interval",
4825                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4826
4827     });
4828 }
4829
4830 sub vm_mon_cmd {
4831     my ($vmid, $execute, %params) = @_;
4832
4833     my $cmd = { execute => $execute, arguments => \%params };
4834     vm_qmp_command($vmid, $cmd);
4835 }
4836
4837 sub vm_mon_cmd_nocheck {
4838     my ($vmid, $execute, %params) = @_;
4839
4840     my $cmd = { execute => $execute, arguments => \%params };
4841     vm_qmp_command($vmid, $cmd, 1);
4842 }
4843
4844 sub vm_qmp_command {
4845     my ($vmid, $cmd, $nocheck) = @_;
4846
4847     my $res;
4848
4849     my $timeout;
4850     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4851         $timeout = $cmd->{arguments}->{timeout};
4852         delete $cmd->{arguments}->{timeout};
4853     }
4854
4855     eval {
4856         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4857         my $sname = qmp_socket($vmid);
4858         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4859             my $qmpclient = PVE::QMPClient->new();
4860
4861             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4862         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4863             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4864                 if scalar(%{$cmd->{arguments}});
4865             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4866         } else {
4867             die "unable to open monitor socket\n";
4868         }
4869     };
4870     if (my $err = $@) {
4871         syslog("err", "VM $vmid qmp command failed - $err");
4872         die $err;
4873     }
4874
4875     return $res;
4876 }
4877
4878 sub vm_human_monitor_command {
4879     my ($vmid, $cmdline) = @_;
4880
4881     my $res;
4882
4883     my $cmd = {
4884         execute => 'human-monitor-command',
4885         arguments => { 'command-line' => $cmdline},
4886     };
4887
4888     return vm_qmp_command($vmid, $cmd);
4889 }
4890
4891 sub vm_commandline {
4892     my ($storecfg, $vmid) = @_;
4893
4894     my $conf = PVE::QemuConfig->load_config($vmid);
4895
4896     my $defaults = load_defaults();
4897
4898     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4899
4900     return PVE::Tools::cmd2string($cmd);
4901 }
4902
4903 sub vm_reset {
4904     my ($vmid, $skiplock) = @_;
4905
4906     PVE::QemuConfig->lock_config($vmid, sub {
4907
4908         my $conf = PVE::QemuConfig->load_config($vmid);
4909
4910         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4911
4912         vm_mon_cmd($vmid, "system_reset");
4913     });
4914 }
4915
4916 sub get_vm_volumes {
4917     my ($conf) = @_;
4918
4919     my $vollist = [];
4920     foreach_volid($conf, sub {
4921         my ($volid, $attr) = @_;
4922
4923         return if $volid =~ m|^/|;
4924
4925         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4926         return if !$sid;
4927
4928         push @$vollist, $volid;
4929     });
4930
4931     return $vollist;
4932 }
4933
4934 sub vm_stop_cleanup {
4935     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4936
4937     eval {
4938
4939         if (!$keepActive) {
4940             my $vollist = get_vm_volumes($conf);
4941             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4942         }
4943
4944         foreach my $ext (qw(mon qmp pid vnc qga)) {
4945             unlink "/var/run/qemu-server/${vmid}.$ext";
4946         }
4947
4948         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4949     };
4950     warn $@ if $@; # avoid errors - just warn
4951 }
4952
4953 # Note: use $nockeck to skip tests if VM configuration file exists.
4954 # We need that when migration VMs to other nodes (files already moved)
4955 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4956 sub vm_stop {
4957     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4958
4959     $force = 1 if !defined($force) && !$shutdown;
4960
4961     if ($migratedfrom){
4962         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4963         kill 15, $pid if $pid;
4964         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4965         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4966         return;
4967     }
4968
4969     PVE::QemuConfig->lock_config($vmid, sub {
4970
4971         my $pid = check_running($vmid, $nocheck);
4972         return if !$pid;
4973
4974         my $conf;
4975         if (!$nocheck) {
4976             $conf = PVE::QemuConfig->load_config($vmid);
4977             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4978             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4979                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4980                 $timeout = $opts->{down} if $opts->{down};
4981             }
4982         }
4983
4984         $timeout = 60 if !defined($timeout);
4985
4986         eval {
4987             if ($shutdown) {
4988                 if (defined($conf) && $conf->{agent}) {
4989                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
4990                 } else {
4991                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
4992                 }
4993             } else {
4994                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
4995             }
4996         };
4997         my $err = $@;
4998
4999         if (!$err) {
5000             my $count = 0;
5001             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5002                 $count++;
5003                 sleep 1;
5004             }
5005
5006             if ($count >= $timeout) {
5007                 if ($force) {
5008                     warn "VM still running - terminating now with SIGTERM\n";
5009                     kill 15, $pid;
5010                 } else {
5011                     die "VM quit/powerdown failed - got timeout\n";
5012                 }
5013             } else {
5014                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5015                 return;
5016             }
5017         } else {
5018             if ($force) {
5019                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5020                 kill 15, $pid;
5021             } else {
5022                 die "VM quit/powerdown failed\n";
5023             }
5024         }
5025
5026         # wait again
5027         $timeout = 10;
5028
5029         my $count = 0;
5030         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5031             $count++;
5032             sleep 1;
5033         }
5034
5035         if ($count >= $timeout) {
5036             warn "VM still running - terminating now with SIGKILL\n";
5037             kill 9, $pid;
5038             sleep 1;
5039         }
5040
5041         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5042    });
5043 }
5044
5045 sub vm_suspend {
5046     my ($vmid, $skiplock) = @_;
5047
5048     PVE::QemuConfig->lock_config($vmid, sub {
5049
5050         my $conf = PVE::QemuConfig->load_config($vmid);
5051
5052         PVE::QemuConfig->check_lock($conf)
5053             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5054
5055         vm_mon_cmd($vmid, "stop");
5056     });
5057 }
5058
5059 sub vm_resume {
5060     my ($vmid, $skiplock, $nocheck) = @_;
5061
5062     PVE::QemuConfig->lock_config($vmid, sub {
5063
5064         if (!$nocheck) {
5065
5066             my $conf = PVE::QemuConfig->load_config($vmid);
5067
5068             PVE::QemuConfig->check_lock($conf)
5069                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5070
5071             vm_mon_cmd($vmid, "cont");
5072
5073         } else {
5074             vm_mon_cmd_nocheck($vmid, "cont");
5075         }
5076     });
5077 }
5078
5079 sub vm_sendkey {
5080     my ($vmid, $skiplock, $key) = @_;
5081
5082     PVE::QemuConfig->lock_config($vmid, sub {
5083
5084         my $conf = PVE::QemuConfig->load_config($vmid);
5085
5086         # there is no qmp command, so we use the human monitor command
5087         vm_human_monitor_command($vmid, "sendkey $key");
5088     });
5089 }
5090
5091 sub vm_destroy {
5092     my ($storecfg, $vmid, $skiplock) = @_;
5093
5094     PVE::QemuConfig->lock_config($vmid, sub {
5095
5096         my $conf = PVE::QemuConfig->load_config($vmid);
5097
5098         if (!check_running($vmid)) {
5099             destroy_vm($storecfg, $vmid, undef, $skiplock);
5100         } else {
5101             die "VM $vmid is running - destroy failed\n";
5102         }
5103     });
5104 }
5105
5106 # pci helpers
5107
5108 sub file_write {
5109     my ($filename, $buf) = @_;
5110
5111     my $fh = IO::File->new($filename, "w");
5112     return undef if !$fh;
5113
5114     my $res = print $fh $buf;
5115
5116     $fh->close();
5117
5118     return $res;
5119 }
5120
5121 sub pci_device_info {
5122     my ($name) = @_;
5123
5124     my $res;
5125
5126     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5127     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5128
5129     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5130     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5131
5132     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5133     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5134
5135     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5136     return undef if !defined($product) || $product !~ s/^0x//;
5137
5138     $res = {
5139         name => $name,
5140         vendor => $vendor,
5141         product => $product,
5142         domain => $domain,
5143         bus => $bus,
5144         slot => $slot,
5145         func => $func,
5146         irq => $irq,
5147         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5148     };
5149
5150     return $res;
5151 }
5152
5153 sub pci_dev_reset {
5154     my ($dev) = @_;
5155
5156     my $name = $dev->{name};
5157
5158     my $fn = "$pcisysfs/devices/$name/reset";
5159
5160     return file_write($fn, "1");
5161 }
5162
5163 sub pci_dev_bind_to_vfio {
5164     my ($dev) = @_;
5165
5166     my $name = $dev->{name};
5167
5168     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5169
5170     if (!-d $vfio_basedir) {
5171         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5172     }
5173     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5174
5175     my $testdir = "$vfio_basedir/$name";
5176     return 1 if -d $testdir;
5177
5178     my $data = "$dev->{vendor} $dev->{product}";
5179     return undef if !file_write("$vfio_basedir/new_id", $data);
5180
5181     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5182     if (!file_write($fn, $name)) {
5183         return undef if -f $fn;
5184     }
5185
5186     $fn = "$vfio_basedir/bind";
5187     if (! -d $testdir) {
5188         return undef if !file_write($fn, $name);
5189     }
5190
5191     return -d $testdir;
5192 }
5193
5194 sub pci_dev_group_bind_to_vfio {
5195     my ($pciid) = @_;
5196
5197     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5198
5199     if (!-d $vfio_basedir) {
5200         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5201     }
5202     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5203
5204     # get IOMMU group devices
5205     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5206       my @devs = grep /^0000:/, readdir($D);
5207     closedir($D);
5208
5209     foreach my $pciid (@devs) {
5210         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5211
5212         # pci bridges, switches or root ports are not supported
5213         # they have a pci_bus subdirectory so skip them
5214         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5215
5216         my $info = pci_device_info($1);
5217         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5218     }
5219
5220     return 1;
5221 }
5222
5223 # vzdump restore implementaion
5224
5225 sub tar_archive_read_firstfile {
5226     my $archive = shift;
5227
5228     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5229
5230     # try to detect archive type first
5231     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5232         die "unable to open file '$archive'\n";
5233     my $firstfile = <$fh>;
5234     kill 15, $pid;
5235     close $fh;
5236
5237     die "ERROR: archive contaions no data\n" if !$firstfile;
5238     chomp $firstfile;
5239
5240     return $firstfile;
5241 }
5242
5243 sub tar_restore_cleanup {
5244     my ($storecfg, $statfile) = @_;
5245
5246     print STDERR "starting cleanup\n";
5247
5248     if (my $fd = IO::File->new($statfile, "r")) {
5249         while (defined(my $line = <$fd>)) {
5250             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5251                 my $volid = $2;
5252                 eval {
5253                     if ($volid =~ m|^/|) {
5254                         unlink $volid || die 'unlink failed\n';
5255                     } else {
5256                         PVE::Storage::vdisk_free($storecfg, $volid);
5257                     }
5258                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5259                 };
5260                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5261             } else {
5262                 print STDERR "unable to parse line in statfile - $line";
5263             }
5264         }
5265         $fd->close();
5266     }
5267 }
5268
5269 sub restore_archive {
5270     my ($archive, $vmid, $user, $opts) = @_;
5271
5272     my $format = $opts->{format};
5273     my $comp;
5274
5275     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5276         $format = 'tar' if !$format;
5277         $comp = 'gzip';
5278     } elsif ($archive =~ m/\.tar$/) {
5279         $format = 'tar' if !$format;
5280     } elsif ($archive =~ m/.tar.lzo$/) {
5281         $format = 'tar' if !$format;
5282         $comp = 'lzop';
5283     } elsif ($archive =~ m/\.vma$/) {
5284         $format = 'vma' if !$format;
5285     } elsif ($archive =~ m/\.vma\.gz$/) {
5286         $format = 'vma' if !$format;
5287         $comp = 'gzip';
5288     } elsif ($archive =~ m/\.vma\.lzo$/) {
5289         $format = 'vma' if !$format;
5290         $comp = 'lzop';
5291     } else {
5292         $format = 'vma' if !$format; # default
5293     }
5294
5295     # try to detect archive format
5296     if ($format eq 'tar') {
5297         return restore_tar_archive($archive, $vmid, $user, $opts);
5298     } else {
5299         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5300     }
5301 }
5302
5303 sub restore_update_config_line {
5304     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5305
5306     return if $line =~ m/^\#qmdump\#/;
5307     return if $line =~ m/^\#vzdump\#/;
5308     return if $line =~ m/^lock:/;
5309     return if $line =~ m/^unused\d+:/;
5310     return if $line =~ m/^parent:/;
5311     return if $line =~ m/^template:/; # restored VM is never a template
5312
5313     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5314     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5315         # try to convert old 1.X settings
5316         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5317         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5318             my ($model, $macaddr) = split(/\=/, $devconfig);
5319             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5320             my $net = {
5321                 model => $model,
5322                 bridge => "vmbr$ind",
5323                 macaddr => $macaddr,
5324             };
5325             my $netstr = print_net($net);
5326
5327             print $outfd "net$cookie->{netcount}: $netstr\n";
5328             $cookie->{netcount}++;
5329         }
5330     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5331         my ($id, $netstr) = ($1, $2);
5332         my $net = parse_net($netstr);
5333         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5334         $netstr = print_net($net);
5335         print $outfd "$id: $netstr\n";
5336     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5337         my $virtdev = $1;
5338         my $value = $3;
5339         my $di = parse_drive($virtdev, $value);
5340         if (defined($di->{backup}) && !$di->{backup}) {
5341             print $outfd "#$line";
5342         } elsif ($map->{$virtdev}) {
5343             delete $di->{format}; # format can change on restore
5344             $di->{file} = $map->{$virtdev};
5345             $value = print_drive($vmid, $di);
5346             print $outfd "$virtdev: $value\n";
5347         } else {
5348             print $outfd $line;
5349         }
5350     } else {
5351         print $outfd $line;
5352     }
5353 }
5354
5355 sub scan_volids {
5356     my ($cfg, $vmid) = @_;
5357
5358     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5359
5360     my $volid_hash = {};
5361     foreach my $storeid (keys %$info) {
5362         foreach my $item (@{$info->{$storeid}}) {
5363             next if !($item->{volid} && $item->{size});
5364             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5365             $volid_hash->{$item->{volid}} = $item;
5366         }
5367     }
5368
5369     return $volid_hash;
5370 }
5371
5372 sub is_volume_in_use {
5373     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5374
5375     my $path = PVE::Storage::path($storecfg, $volid);
5376
5377     my $scan_config = sub {
5378         my ($cref, $snapname) = @_;
5379
5380         foreach my $key (keys %$cref) {
5381             my $value = $cref->{$key};
5382             if (is_valid_drivename($key)) {
5383                 next if $skip_drive && $key eq $skip_drive;
5384                 my $drive = parse_drive($key, $value);
5385                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5386                 return 1 if $volid eq $drive->{file};
5387                 if ($drive->{file} =~ m!^/!) {
5388                     return 1 if $drive->{file} eq $path;
5389                 } else {
5390                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5391                     next if !$storeid;
5392                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5393                     next if !$scfg;
5394                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5395                 }
5396             }
5397         }
5398
5399         return 0;
5400     };
5401
5402     return 1 if &$scan_config($conf);
5403
5404     undef $skip_drive;
5405
5406     foreach my $snapname (keys %{$conf->{snapshots}}) {
5407         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5408     }
5409
5410     return 0;
5411 }
5412
5413 sub update_disksize {
5414     my ($vmid, $conf, $volid_hash) = @_;
5415
5416     my $changes;
5417
5418     # used and unused disks
5419     my $referenced = {};
5420
5421     # Note: it is allowed to define multiple storages with same path (alias), so
5422     # we need to check both 'volid' and real 'path' (two different volid can point
5423     # to the same path).
5424
5425     my $referencedpath = {};
5426
5427     # update size info
5428     foreach my $opt (keys %$conf) {
5429         if (is_valid_drivename($opt)) {
5430             my $drive = parse_drive($opt, $conf->{$opt});
5431             my $volid = $drive->{file};
5432             next if !$volid;
5433
5434             $referenced->{$volid} = 1;
5435             if ($volid_hash->{$volid} &&
5436                 (my $path = $volid_hash->{$volid}->{path})) {
5437                 $referencedpath->{$path} = 1;
5438             }
5439
5440             next if drive_is_cdrom($drive);
5441             next if !$volid_hash->{$volid};
5442
5443             $drive->{size} = $volid_hash->{$volid}->{size};
5444             my $new = print_drive($vmid, $drive);
5445             if ($new ne $conf->{$opt}) {
5446                 $changes = 1;
5447                 $conf->{$opt} = $new;
5448             }
5449         }
5450     }
5451
5452     # remove 'unusedX' entry if volume is used
5453     foreach my $opt (keys %$conf) {
5454         next if $opt !~ m/^unused\d+$/;
5455         my $volid = $conf->{$opt};
5456         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5457         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5458             $changes = 1;
5459             delete $conf->{$opt};
5460         }
5461
5462         $referenced->{$volid} = 1;
5463         $referencedpath->{$path} = 1 if $path;
5464     }
5465
5466     foreach my $volid (sort keys %$volid_hash) {
5467         next if $volid =~ m/vm-$vmid-state-/;
5468         next if $referenced->{$volid};
5469         my $path = $volid_hash->{$volid}->{path};
5470         next if !$path; # just to be sure
5471         next if $referencedpath->{$path};
5472         $changes = 1;
5473         PVE::QemuConfig->add_unused_volume($conf, $volid);
5474         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5475     }
5476
5477     return $changes;
5478 }
5479
5480 sub rescan {
5481     my ($vmid, $nolock) = @_;
5482
5483     my $cfg = PVE::Storage::config();
5484
5485     my $volid_hash = scan_volids($cfg, $vmid);
5486
5487     my $updatefn =  sub {
5488         my ($vmid) = @_;
5489
5490         my $conf = PVE::QemuConfig->load_config($vmid);
5491
5492         PVE::QemuConfig->check_lock($conf);
5493
5494         my $vm_volids = {};
5495         foreach my $volid (keys %$volid_hash) {
5496             my $info = $volid_hash->{$volid};
5497             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5498         }
5499
5500         my $changes = update_disksize($vmid, $conf, $vm_volids);
5501
5502         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5503     };
5504
5505     if (defined($vmid)) {
5506         if ($nolock) {
5507             &$updatefn($vmid);
5508         } else {
5509             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5510         }
5511     } else {
5512         my $vmlist = config_list();
5513         foreach my $vmid (keys %$vmlist) {
5514             if ($nolock) {
5515                 &$updatefn($vmid);
5516             } else {
5517                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5518             }
5519         }
5520     }
5521 }
5522
5523 sub restore_vma_archive {
5524     my ($archive, $vmid, $user, $opts, $comp) = @_;
5525
5526     my $input = $archive eq '-' ? "<&STDIN" : undef;
5527     my $readfrom = $archive;
5528
5529     my $uncomp = '';
5530     if ($comp) {
5531         $readfrom = '-';
5532         my $qarchive = PVE::Tools::shellquote($archive);
5533         if ($comp eq 'gzip') {
5534             $uncomp = "zcat $qarchive|";
5535         } elsif ($comp eq 'lzop') {
5536             $uncomp = "lzop -d -c $qarchive|";
5537         } else {
5538             die "unknown compression method '$comp'\n";
5539         }
5540
5541     }
5542
5543     my $tmpdir = "/var/tmp/vzdumptmp$$";
5544     rmtree $tmpdir;
5545
5546     # disable interrupts (always do cleanups)
5547     local $SIG{INT} =
5548         local $SIG{TERM} =
5549         local $SIG{QUIT} =
5550         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5551
5552     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5553     POSIX::mkfifo($mapfifo, 0600);
5554     my $fifofh;
5555
5556     my $openfifo = sub {
5557         open($fifofh, '>', $mapfifo) || die $!;
5558     };
5559
5560     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5561
5562     my $oldtimeout;
5563     my $timeout = 5;
5564
5565     my $devinfo = {};
5566
5567     my $rpcenv = PVE::RPCEnvironment::get();
5568
5569     my $conffile = PVE::QemuConfig->config_file($vmid);
5570     my $tmpfn = "$conffile.$$.tmp";
5571
5572     # Note: $oldconf is undef if VM does not exists
5573     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5574     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5575
5576     my $print_devmap = sub {
5577         my $virtdev_hash = {};
5578
5579         my $cfgfn = "$tmpdir/qemu-server.conf";
5580
5581         # we can read the config - that is already extracted
5582         my $fh = IO::File->new($cfgfn, "r") ||
5583             "unable to read qemu-server.conf - $!\n";
5584
5585         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5586         if (-f $fwcfgfn) {
5587             my $pve_firewall_dir = '/etc/pve/firewall';
5588             mkdir $pve_firewall_dir; # make sure the dir exists
5589             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5590         }
5591
5592         while (defined(my $line = <$fh>)) {
5593             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5594                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5595                 die "archive does not contain data for drive '$virtdev'\n"
5596                     if !$devinfo->{$devname};
5597                 if (defined($opts->{storage})) {
5598                     $storeid = $opts->{storage} || 'local';
5599                 } elsif (!$storeid) {
5600                     $storeid = 'local';
5601                 }
5602                 $format = 'raw' if !$format;
5603                 $devinfo->{$devname}->{devname} = $devname;
5604                 $devinfo->{$devname}->{virtdev} = $virtdev;
5605                 $devinfo->{$devname}->{format} = $format;
5606                 $devinfo->{$devname}->{storeid} = $storeid;
5607
5608                 # check permission on storage
5609                 my $pool = $opts->{pool}; # todo: do we need that?
5610                 if ($user ne 'root@pam') {
5611                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5612                 }
5613
5614                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5615             }
5616         }
5617
5618         foreach my $devname (keys %$devinfo) {
5619             die "found no device mapping information for device '$devname'\n"
5620                 if !$devinfo->{$devname}->{virtdev};
5621         }
5622
5623         my $cfg = PVE::Storage::config();
5624
5625         # create empty/temp config
5626         if ($oldconf) {
5627             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5628             foreach_drive($oldconf, sub {
5629                 my ($ds, $drive) = @_;
5630
5631                 return if drive_is_cdrom($drive);
5632
5633                 my $volid = $drive->{file};
5634
5635                 return if !$volid || $volid =~ m|^/|;
5636
5637                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5638                 return if !$path || !$owner || ($owner != $vmid);
5639
5640                 # Note: only delete disk we want to restore
5641                 # other volumes will become unused
5642                 if ($virtdev_hash->{$ds}) {
5643                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5644                     if (my $err = $@) {
5645                         warn $err;
5646                     }
5647                 }
5648             });
5649
5650             # delete vmstate files
5651             # since after the restore we have no snapshots anymore
5652             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5653                 my $snap = $oldconf->{snapshots}->{$snapname};
5654                 if ($snap->{vmstate}) {
5655                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5656                     if (my $err = $@) {
5657                         warn $err;
5658                     }
5659                 }
5660             }
5661         }
5662
5663         my $map = {};
5664         foreach my $virtdev (sort keys %$virtdev_hash) {
5665             my $d = $virtdev_hash->{$virtdev};
5666             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5667             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5668
5669             # test if requested format is supported
5670             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5671             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5672             $d->{format} = $defFormat if !$supported;
5673
5674             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5675                                                   $d->{format}, undef, $alloc_size);
5676             print STDERR "new volume ID is '$volid'\n";
5677             $d->{volid} = $volid;
5678             my $path = PVE::Storage::path($cfg, $volid);
5679
5680             PVE::Storage::activate_volumes($cfg,[$volid]);
5681
5682             my $write_zeros = 1;
5683             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5684                 $write_zeros = 0;
5685             }
5686
5687             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5688
5689             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5690             $map->{$virtdev} = $volid;
5691         }
5692
5693         $fh->seek(0, 0) || die "seek failed - $!\n";
5694
5695         my $outfd = new IO::File ($tmpfn, "w") ||
5696             die "unable to write config for VM $vmid\n";
5697
5698         my $cookie = { netcount => 0 };
5699         while (defined(my $line = <$fh>)) {
5700             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5701         }
5702
5703         $fh->close();
5704         $outfd->close();
5705     };
5706
5707     eval {
5708         # enable interrupts
5709         local $SIG{INT} =
5710             local $SIG{TERM} =
5711             local $SIG{QUIT} =
5712             local $SIG{HUP} =
5713             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5714         local $SIG{ALRM} = sub { die "got timeout\n"; };
5715
5716         $oldtimeout = alarm($timeout);
5717
5718         my $parser = sub {
5719             my $line = shift;
5720
5721             print "$line\n";
5722
5723             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5724                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5725                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5726             } elsif ($line =~ m/^CTIME: /) {
5727                 # we correctly received the vma config, so we can disable
5728                 # the timeout now for disk allocation (set to 10 minutes, so
5729                 # that we always timeout if something goes wrong)
5730                 alarm(600);
5731                 &$print_devmap();
5732                 print $fifofh "done\n";
5733                 my $tmp = $oldtimeout || 0;
5734                 $oldtimeout = undef;
5735                 alarm($tmp);
5736                 close($fifofh);
5737             }
5738         };
5739
5740         print "restore vma archive: $cmd\n";
5741         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5742     };
5743     my $err = $@;
5744
5745     alarm($oldtimeout) if $oldtimeout;
5746
5747     my $vollist = [];
5748     foreach my $devname (keys %$devinfo) {
5749         my $volid = $devinfo->{$devname}->{volid};
5750         push @$vollist, $volid if $volid;
5751     }
5752
5753     my $cfg = PVE::Storage::config();
5754     PVE::Storage::deactivate_volumes($cfg, $vollist);
5755
5756     unlink $mapfifo;
5757
5758     if ($err) {
5759         rmtree $tmpdir;
5760         unlink $tmpfn;
5761
5762         foreach my $devname (keys %$devinfo) {
5763             my $volid = $devinfo->{$devname}->{volid};
5764             next if !$volid;
5765             eval {
5766                 if ($volid =~ m|^/|) {
5767                     unlink $volid || die 'unlink failed\n';
5768                 } else {
5769                     PVE::Storage::vdisk_free($cfg, $volid);
5770                 }
5771                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5772             };
5773             print STDERR "unable to cleanup '$volid' - $@" if $@;
5774         }
5775         die $err;
5776     }
5777
5778     rmtree $tmpdir;
5779
5780     rename($tmpfn, $conffile) ||
5781         die "unable to commit configuration file '$conffile'\n";
5782
5783     PVE::Cluster::cfs_update(); # make sure we read new file
5784
5785     eval { rescan($vmid, 1); };
5786     warn $@ if $@;
5787 }
5788
5789 sub restore_tar_archive {
5790     my ($archive, $vmid, $user, $opts) = @_;
5791
5792     if ($archive ne '-') {
5793         my $firstfile = tar_archive_read_firstfile($archive);
5794         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5795             if $firstfile ne 'qemu-server.conf';
5796     }
5797
5798     my $storecfg = PVE::Storage::config();
5799
5800     # destroy existing data - keep empty config
5801     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5802     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5803
5804     my $tocmd = "/usr/lib/qemu-server/qmextract";
5805
5806     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5807     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5808     $tocmd .= ' --prealloc' if $opts->{prealloc};
5809     $tocmd .= ' --info' if $opts->{info};
5810
5811     # tar option "xf" does not autodetect compression when read from STDIN,
5812     # so we pipe to zcat
5813     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5814         PVE::Tools::shellquote("--to-command=$tocmd");
5815
5816     my $tmpdir = "/var/tmp/vzdumptmp$$";
5817     mkpath $tmpdir;
5818
5819     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5820     local $ENV{VZDUMP_VMID} = $vmid;
5821     local $ENV{VZDUMP_USER} = $user;
5822
5823     my $conffile = PVE::QemuConfig->config_file($vmid);
5824     my $tmpfn = "$conffile.$$.tmp";
5825
5826     # disable interrupts (always do cleanups)
5827     local $SIG{INT} =
5828         local $SIG{TERM} =
5829         local $SIG{QUIT} =
5830         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5831
5832     eval {
5833         # enable interrupts
5834         local $SIG{INT} =
5835             local $SIG{TERM} =
5836             local $SIG{QUIT} =
5837             local $SIG{HUP} =
5838             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5839
5840         if ($archive eq '-') {
5841             print "extracting archive from STDIN\n";
5842             run_command($cmd, input => "<&STDIN");
5843         } else {
5844             print "extracting archive '$archive'\n";
5845             run_command($cmd);
5846         }
5847
5848         return if $opts->{info};
5849
5850         # read new mapping
5851         my $map = {};
5852         my $statfile = "$tmpdir/qmrestore.stat";
5853         if (my $fd = IO::File->new($statfile, "r")) {
5854             while (defined (my $line = <$fd>)) {
5855                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5856                     $map->{$1} = $2 if $1;
5857                 } else {
5858                     print STDERR "unable to parse line in statfile - $line\n";
5859                 }
5860             }
5861             $fd->close();
5862         }
5863
5864         my $confsrc = "$tmpdir/qemu-server.conf";
5865
5866         my $srcfd = new IO::File($confsrc, "r") ||
5867             die "unable to open file '$confsrc'\n";
5868
5869         my $outfd = new IO::File ($tmpfn, "w") ||
5870             die "unable to write config for VM $vmid\n";
5871
5872         my $cookie = { netcount => 0 };
5873         while (defined (my $line = <$srcfd>)) {
5874             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5875         }
5876
5877         $srcfd->close();
5878         $outfd->close();
5879     };
5880     my $err = $@;
5881
5882     if ($err) {
5883
5884         unlink $tmpfn;
5885
5886         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5887
5888         die $err;
5889     }
5890
5891     rmtree $tmpdir;
5892
5893     rename $tmpfn, $conffile ||
5894         die "unable to commit configuration file '$conffile'\n";
5895
5896     PVE::Cluster::cfs_update(); # make sure we read new file
5897
5898     eval { rescan($vmid, 1); };
5899     warn $@ if $@;
5900 };
5901
5902 sub foreach_storage_used_by_vm {
5903     my ($conf, $func) = @_;
5904
5905     my $sidhash = {};
5906
5907     foreach_drive($conf, sub {
5908         my ($ds, $drive) = @_;
5909         return if drive_is_cdrom($drive);
5910
5911         my $volid = $drive->{file};
5912
5913         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5914         $sidhash->{$sid} = $sid if $sid;
5915     });
5916
5917     foreach my $sid (sort keys %$sidhash) {
5918         &$func($sid);
5919     }
5920 }
5921
5922 sub do_snapshots_with_qemu {
5923     my ($storecfg, $volid) = @_;
5924
5925     my $storage_name = PVE::Storage::parse_volume_id($volid);
5926
5927     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5928         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5929         return 1;
5930     }
5931
5932     if ($volid =~ m/\.(qcow2|qed)$/){
5933         return 1;
5934     }
5935
5936     return undef;
5937 }
5938
5939 sub qga_check_running {
5940     my ($vmid) = @_;
5941
5942     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5943     if ($@) {
5944         warn "Qemu Guest Agent is not running - $@";
5945         return 0;
5946     }
5947     return 1;
5948 }
5949
5950 sub template_create {
5951     my ($vmid, $conf, $disk) = @_;
5952
5953     my $storecfg = PVE::Storage::config();
5954
5955     foreach_drive($conf, sub {
5956         my ($ds, $drive) = @_;
5957
5958         return if drive_is_cdrom($drive);
5959         return if $disk && $ds ne $disk;
5960
5961         my $volid = $drive->{file};
5962         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5963
5964         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5965         $drive->{file} = $voliddst;
5966         $conf->{$ds} = print_drive($vmid, $drive);
5967         PVE::QemuConfig->write_config($vmid, $conf);
5968     });
5969 }
5970
5971 sub qemu_img_convert {
5972     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5973
5974     my $storecfg = PVE::Storage::config();
5975     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
5976     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
5977
5978     if ($src_storeid && $dst_storeid) {
5979
5980         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
5981
5982         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
5983         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
5984
5985         my $src_format = qemu_img_format($src_scfg, $src_volname);
5986         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
5987
5988         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
5989         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
5990
5991         my $cmd = [];
5992         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
5993         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
5994         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
5995         if ($is_zero_initialized) {
5996             push @$cmd, "zeroinit:$dst_path";
5997         } else {
5998             push @$cmd, $dst_path;
5999         }
6000
6001         my $parser = sub {
6002             my $line = shift;
6003             if($line =~ m/\((\S+)\/100\%\)/){
6004                 my $percent = $1;
6005                 my $transferred = int($size * $percent / 100);
6006                 my $remaining = $size - $transferred;
6007
6008                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6009             }
6010
6011         };
6012
6013         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6014         my $err = $@;
6015         die "copy failed: $err" if $err;
6016     }
6017 }
6018
6019 sub qemu_img_format {
6020     my ($scfg, $volname) = @_;
6021
6022     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6023         return $1;
6024     } else {
6025         return "raw";
6026     }
6027 }
6028
6029 sub qemu_drive_mirror {
6030     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6031
6032     $jobs = {} if !$jobs;
6033
6034     my $qemu_target;
6035     my $format;
6036     $jobs->{"drive-$drive"} = {};
6037
6038     if ($dst_volid =~ /^nbd:(localhost|[\d\.]+|\[[\d\.:a-fA-F]+\]):(\d+):exportname=(\S+)/) {
6039         my $server = $1;
6040         my $port = $2;
6041         my $exportname = $3;
6042
6043         $format = "nbd";
6044         my $unixsocket = "/run/qemu-server/$vmid.mirror-drive-$drive";
6045         $qemu_target = "nbd+unix:///$exportname?socket=$unixsocket";
6046         my $cmd = ['socat', '-T30', "UNIX-LISTEN:$unixsocket,fork", "TCP:$server:$2,connect-timeout=5"];
6047
6048         my $pid = fork();
6049         if (!defined($pid)) {
6050             die "forking socat tunnel failed\n";
6051         } elsif ($pid == 0) {
6052             exec(@$cmd);
6053             warn "exec failed: $!\n";
6054             POSIX::_exit(-1);
6055         }
6056         $jobs->{"drive-$drive"}->{pid} = $pid;
6057
6058         my $timeout = 0;
6059         while (!-S $unixsocket) {
6060             die "nbd connection helper timed out\n"
6061                 if $timeout++ > 5;
6062             sleep 1;
6063         }
6064     } else {
6065         my $storecfg = PVE::Storage::config();
6066         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6067
6068         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6069
6070         $format = qemu_img_format($dst_scfg, $dst_volname);
6071
6072         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6073
6074         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6075     }
6076
6077     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6078     $opts->{format} = $format if $format;
6079
6080     print "drive mirror is starting for drive-$drive\n";
6081
6082     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6083
6084     if (my $err = $@) {
6085         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6086         die "mirroring error: $err";
6087     }
6088
6089     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6090 }
6091
6092 sub qemu_drive_mirror_monitor {
6093     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6094
6095     eval {
6096         my $err_complete = 0;
6097
6098         while (1) {
6099             die "storage migration timed out\n" if $err_complete > 300;
6100
6101             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6102
6103             my $running_mirror_jobs = {};
6104             foreach my $stat (@$stats) {
6105                 next if $stat->{type} ne 'mirror';
6106                 $running_mirror_jobs->{$stat->{device}} = $stat;
6107             }
6108
6109             my $readycounter = 0;
6110
6111             foreach my $job (keys %$jobs) {
6112
6113                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6114                     print "$job : finished\n";
6115                     delete $jobs->{$job};
6116                     next;
6117                 }
6118
6119                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6120
6121                 my $busy = $running_mirror_jobs->{$job}->{busy};
6122                 my $ready = $running_mirror_jobs->{$job}->{ready};
6123                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6124                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6125                     my $remaining = $total - $transferred;
6126                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6127
6128                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6129                 }
6130
6131                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6132             }
6133
6134             last if scalar(keys %$jobs) == 0;
6135
6136             if ($readycounter == scalar(keys %$jobs)) {
6137                 print "all mirroring jobs are ready \n";
6138                 last if $skipcomplete; #do the complete later
6139
6140                 if ($vmiddst && $vmiddst != $vmid) {
6141                     my $agent_running = $qga && qga_check_running($vmid);
6142                     if ($agent_running) {
6143                         print "freeze filesystem\n";
6144                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6145                     } else {
6146                         print "suspend vm\n";
6147                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6148                     }
6149
6150                     # if we clone a disk for a new target vm, we don't switch the disk
6151                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6152
6153                     if ($agent_running) {
6154                         print "unfreeze filesystem\n";
6155                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6156                     } else {
6157                         print "resume vm\n";
6158                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6159                     }
6160
6161                     last;
6162                 } else {
6163
6164                     foreach my $job (keys %$jobs) {
6165                         # try to switch the disk if source and destination are on the same guest
6166                         print "$job: Completing block job...\n";
6167
6168                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6169                         if ($@ =~ m/cannot be completed/) {
6170                             print "$job: Block job cannot be completed, try again.\n";
6171                             $err_complete++;
6172                         }else {
6173                             print "$job: Completed successfully.\n";
6174                             $jobs->{$job}->{complete} = 1;
6175                             eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6176                         }
6177                     }
6178                 }
6179             }
6180             sleep 1;
6181         }
6182     };
6183     my $err = $@;
6184
6185     if ($err) {
6186         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6187         die "mirroring error: $err";
6188     }
6189
6190 }
6191
6192 sub qemu_blockjobs_cancel {
6193     my ($vmid, $jobs) = @_;
6194
6195     foreach my $job (keys %$jobs) {
6196         print "$job: Cancelling block job\n";
6197         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6198         $jobs->{$job}->{cancel} = 1;
6199     }
6200
6201     while (1) {
6202         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6203
6204         my $running_jobs = {};
6205         foreach my $stat (@$stats) {
6206             $running_jobs->{$stat->{device}} = $stat;
6207         }
6208
6209         foreach my $job (keys %$jobs) {
6210
6211             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6212                 print "$job: Done.\n";
6213                 eval { qemu_blockjobs_finish_tunnel($vmid, $job, $jobs->{$job}->{pid}) } ;
6214                 delete $jobs->{$job};
6215             }
6216         }
6217
6218         last if scalar(keys %$jobs) == 0;
6219
6220         sleep 1;
6221     }
6222 }
6223
6224 sub qemu_blockjobs_finish_tunnel {
6225    my ($vmid, $job, $cpid) = @_;
6226
6227    return if !$cpid;
6228
6229    for (my $i = 1; $i < 20; $i++) {
6230         my $waitpid = waitpid($cpid, WNOHANG);
6231         last if (defined($waitpid) && ($waitpid == $cpid));
6232
6233         if ($i == 10) {
6234             kill(15, $cpid);
6235         } elsif ($i >= 15) {
6236             kill(9, $cpid);
6237         }
6238         sleep (1);
6239     }
6240     unlink "/run/qemu-server/$vmid.mirror-$job";
6241 }
6242
6243 sub clone_disk {
6244     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6245         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6246
6247     my $newvolid;
6248
6249     if (!$full) {
6250         print "create linked clone of drive $drivename ($drive->{file})\n";
6251         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6252         push @$newvollist, $newvolid;
6253     } else {
6254
6255         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6256         $storeid = $storage if $storage;
6257
6258         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6259         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6260
6261         print "create full clone of drive $drivename ($drive->{file})\n";
6262         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, undef, ($size/1024));
6263         push @$newvollist, $newvolid;
6264
6265         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6266
6267         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6268         if (!$running || $snapname) {
6269             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6270         } else {
6271
6272             my $kvmver = get_running_qemu_version ($vmid);
6273             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6274                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6275                     if $drive->{iothread};
6276             }
6277
6278             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6279         }
6280     }
6281
6282     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6283
6284     my $disk = $drive;
6285     $disk->{format} = undef;
6286     $disk->{file} = $newvolid;
6287     $disk->{size} = $size;
6288
6289     return $disk;
6290 }
6291
6292 # this only works if VM is running
6293 sub get_current_qemu_machine {
6294     my ($vmid) = @_;
6295
6296     my $cmd = { execute => 'query-machines', arguments => {} };
6297     my $res = vm_qmp_command($vmid, $cmd);
6298
6299     my ($current, $default);
6300     foreach my $e (@$res) {
6301         $default = $e->{name} if $e->{'is-default'};
6302         $current = $e->{name} if $e->{'is-current'};
6303     }
6304
6305     # fallback to the default machine if current is not supported by qemu
6306     return $current || $default || 'pc';
6307 }
6308
6309 sub get_running_qemu_version {
6310     my ($vmid) = @_;
6311     my $cmd = { execute => 'query-version', arguments => {} };
6312     my $res = vm_qmp_command($vmid, $cmd);
6313     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6314 }
6315
6316 sub qemu_machine_feature_enabled {
6317     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6318
6319     my $current_major;
6320     my $current_minor;
6321
6322     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6323
6324         $current_major = $3;
6325         $current_minor = $4;
6326
6327     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6328
6329         $current_major = $1;
6330         $current_minor = $2;
6331     }
6332
6333     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6334
6335
6336 }
6337
6338 sub qemu_machine_pxe {
6339     my ($vmid, $conf, $machine) = @_;
6340
6341     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6342
6343     foreach my $opt (keys %$conf) {
6344         next if $opt !~ m/^net(\d+)$/;
6345         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6346         next if !$net;
6347         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6348         return $machine.".pxe" if $romfile =~ m/pxe/;
6349         last;
6350     }
6351
6352     return $machine;
6353 }
6354
6355 sub qemu_use_old_bios_files {
6356     my ($machine_type) = @_;
6357
6358     return if !$machine_type;
6359
6360     my $use_old_bios_files = undef;
6361
6362     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6363         $machine_type = $1;
6364         $use_old_bios_files = 1;
6365     } else {
6366         my $kvmver = kvm_user_version();
6367         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6368         # load new efi bios files on migration. So this hack is required to allow
6369         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6370         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6371         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6372     }
6373
6374     return ($use_old_bios_files, $machine_type);
6375 }
6376
6377 sub create_efidisk {
6378     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6379
6380     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6381
6382     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6383     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6384     PVE::Storage::activate_volumes($storecfg, [$volid]);
6385
6386     my $path = PVE::Storage::path($storecfg, $volid);
6387     eval {
6388         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6389     };
6390     die "Copying EFI vars image failed: $@" if $@;
6391
6392     return ($volid, $vars_size);
6393 }
6394
6395 sub lspci {
6396
6397     my $devices = {};
6398
6399     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6400             my (undef, $id, $function) = @_;
6401             my $res = { id => $id, function => $function};
6402             push @{$devices->{$id}}, $res;
6403     });
6404
6405     # Entries should be sorted by functions.
6406     foreach my $id (keys %$devices) {
6407         my $dev = $devices->{$id};
6408         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6409     }
6410
6411     return $devices;
6412 }
6413
6414 sub vm_iothreads_list {
6415     my ($vmid) = @_;
6416
6417     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6418
6419     my $iothreads = {};
6420     foreach my $iothread (@$res) {
6421         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6422     }
6423
6424     return $iothreads;
6425 }
6426
6427 sub scsihw_infos {
6428     my ($conf, $drive) = @_;
6429
6430     my $maxdev = 0;
6431
6432     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6433         $maxdev = 7;
6434     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6435         $maxdev = 1;
6436     } else {
6437         $maxdev = 256;
6438     }
6439
6440     my $controller = int($drive->{index} / $maxdev);
6441     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6442
6443     return ($maxdev, $controller, $controller_prefix);
6444 }
6445
6446 sub add_hyperv_enlightenments {
6447     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6448
6449     return if $winversion < 6;
6450     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6451
6452     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6453
6454     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6455         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6456         push @$cpuFlags , 'hv_vapic';
6457         push @$cpuFlags , 'hv_time';
6458     } else {
6459         push @$cpuFlags , 'hv_spinlocks=0xffff';
6460     }
6461
6462     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6463         push @$cpuFlags , 'hv_reset';
6464         push @$cpuFlags , 'hv_vpindex';
6465         push @$cpuFlags , 'hv_runtime';
6466     }
6467
6468     if ($winversion >= 7) {
6469         push @$cpuFlags , 'hv_relaxed';
6470     }
6471 }
6472
6473 sub windows_version {
6474     my ($ostype) = @_;
6475
6476     return 0 if !$ostype;
6477
6478     my $winversion = 0;
6479
6480     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6481         $winversion = 5;
6482     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6483         $winversion = 6;
6484     } elsif ($ostype =~ m/^win(\d+)$/) {
6485         $winversion = $1;
6486     }
6487
6488     return $winversion;
6489 }
6490
6491 sub resolve_dst_disk_format {
6492         my ($storecfg, $storeid, $src_volname, $format) = @_;
6493         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6494
6495         if (!$format) {
6496             # if no target format is specified, use the source disk format as hint
6497             if ($src_volname) {
6498                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6499                 $format = qemu_img_format($scfg, $src_volname);
6500             } else {
6501                 return $defFormat;
6502             }
6503         }
6504
6505         # test if requested format is supported - else use default
6506         my $supported = grep { $_ eq $format } @$validFormats;
6507         $format = $defFormat if !$supported;
6508         return $format;
6509 }
6510
6511 sub resolve_first_disk {
6512     my $conf = shift;
6513     my @disks = PVE::QemuServer::valid_drive_names();
6514     my $firstdisk;
6515     foreach my $ds (reverse @disks) {
6516         next if !$conf->{$ds};
6517         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6518         next if PVE::QemuServer::drive_is_cdrom($disk);
6519         $firstdisk = $ds;
6520     }
6521     return $firstdisk;
6522 }
6523
6524 sub generate_smbios1_uuid {
6525     my ($uuid, $uuid_str);
6526     UUID::generate($uuid);
6527     UUID::unparse($uuid, $uuid_str);
6528     return "uuid=$uuid_str";
6529 }
6530
6531 # bash completion helper
6532
6533 sub complete_backup_archives {
6534     my ($cmdname, $pname, $cvalue) = @_;
6535
6536     my $cfg = PVE::Storage::config();
6537
6538     my $storeid;
6539
6540     if ($cvalue =~ m/^([^:]+):/) {
6541         $storeid = $1;
6542     }
6543
6544     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6545
6546     my $res = [];
6547     foreach my $id (keys %$data) {
6548         foreach my $item (@{$data->{$id}}) {
6549             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6550             push @$res, $item->{volid} if defined($item->{volid});
6551         }
6552     }
6553
6554     return $res;
6555 }
6556
6557 my $complete_vmid_full = sub {
6558     my ($running) = @_;
6559
6560     my $idlist = vmstatus();
6561
6562     my $res = [];
6563
6564     foreach my $id (keys %$idlist) {
6565         my $d = $idlist->{$id};
6566         if (defined($running)) {
6567             next if $d->{template};
6568             next if $running && $d->{status} ne 'running';
6569             next if !$running && $d->{status} eq 'running';
6570         }
6571         push @$res, $id;
6572
6573     }
6574     return $res;
6575 };
6576
6577 sub complete_vmid {
6578     return &$complete_vmid_full();
6579 }
6580
6581 sub complete_vmid_stopped {
6582     return &$complete_vmid_full(0);
6583 }
6584
6585 sub complete_vmid_running {
6586     return &$complete_vmid_full(1);
6587 }
6588
6589 sub complete_storage {
6590
6591     my $cfg = PVE::Storage::config();
6592     my $ids = $cfg->{ids};
6593
6594     my $res = [];
6595     foreach my $sid (keys %$ids) {
6596         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6597         next if !$ids->{$sid}->{content}->{images};
6598         push @$res, $sid;
6599     }
6600
6601     return $res;
6602 }
6603
6604 sub nbd_stop {
6605     my ($vmid) = @_;
6606
6607     vm_mon_cmd($vmid, 'nbd-server-stop');
6608 }
6609
6610 1;