PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42
  43 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  44
  45 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  46
  47 # Note about locking: we use flock on the config file protect
  48 # against concurent actions.
  49 # Aditionaly, we have a 'lock' setting in the config file. This
  50 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  51 # allowed when such lock is set. But you can ignore this kind of
  52 # lock with the --skiplock flag.
  53
  54 cfs_register_file('/qemu-server/',
  55                   \&parse_vm_config,
  56                   \&write_vm_config);
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  72     type => 'string',
  73     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  74     description => "The drive's backing file's data format.",
  75     optional => 1,
  76 });
  77
  78 #no warnings 'redefine';
  79
  80 sub cgroups_write {
  81    my ($controller, $vmid, $option, $value) = @_;
  82
  83    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  84    PVE::ProcFSTools::write_proc_entry($path, $value);
  85
  86 }
  87
  88 my $nodename = PVE::INotify::nodename();
  89
  90 mkdir "/etc/pve/nodes/$nodename";
  91 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  92 mkdir $confdir;
  93
  94 my $var_run_tmpdir = "/var/run/qemu-server";
  95 mkdir $var_run_tmpdir;
  96
  97 my $lock_dir = "/var/lock/qemu-server";
  98 mkdir $lock_dir;
  99
 100 my $pcisysfs = "/sys/bus/pci";
 101
 102 my $cpu_vendor_list = {
 103     # Intel CPUs
 104     486 => 'GenuineIntel',
 105     pentium => 'GenuineIntel',
 106     pentium2  => 'GenuineIntel',
 107     pentium3  => 'GenuineIntel',
 108     coreduo => 'GenuineIntel',
 109     core2duo => 'GenuineIntel',
 110     Conroe  => 'GenuineIntel',
 111     Penryn  => 'GenuineIntel',
 112     Nehalem  => 'GenuineIntel',
 113     'Nehalem-IBRS'  => 'GenuineIntel',
 114     Westmere => 'GenuineIntel',
 115     'Westmere-IBRS' => 'GenuineIntel',
 116     SandyBridge => 'GenuineIntel',
 117     'SandyBridge-IBRS' => 'GenuineIntel',
 118     IvyBridge => 'GenuineIntel',
 119     'IvyBridge-IBRS' => 'GenuineIntel',
 120     Haswell => 'GenuineIntel',
 121     'Haswell-IBRS' => 'GenuineIntel',
 122     'Haswell-noTSX' => 'GenuineIntel',
 123     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 124     Broadwell => 'GenuineIntel',
 125     'Broadwell-IBRS' => 'GenuineIntel',
 126     'Broadwell-noTSX' => 'GenuineIntel',
 127     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 128     'Skylake-Client' => 'GenuineIntel',
 129     'Skylake-Client-IBRS' => 'GenuineIntel',
 130     'Skylake-Server' => 'GenuineIntel',
 131     'Skylake-Server-IBRS' => 'GenuineIntel',
 132
 133     # AMD CPUs
 134     athlon => 'AuthenticAMD',
 135     phenom  => 'AuthenticAMD',
 136     Opteron_G1  => 'AuthenticAMD',
 137     Opteron_G2  => 'AuthenticAMD',
 138     Opteron_G3  => 'AuthenticAMD',
 139     Opteron_G4  => 'AuthenticAMD',
 140     Opteron_G5  => 'AuthenticAMD',
 141
 142     # generic types, use vendor from host node
 143     host => 'default',
 144     kvm32 => 'default',
 145     kvm64 => 'default',
 146     qemu32 => 'default',
 147     qemu64 => 'default',
 148     max => 'default',
 149 };
 150
 151 my $cpu_flag = qr/[+-](pcid|spec-ctrl)/;
 152
 153 my $cpu_fmt = {
 154     cputype => {
 155         description => "Emulated CPU type.",
 156         type => 'string',
 157         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 158         default => 'kvm64',
 159         default_key => 1,
 160     },
 161     hidden => {
 162         description => "Do not identify as a KVM virtual machine.",
 163         type => 'boolean',
 164         optional => 1,
 165         default => 0
 166     },
 167     flags => {
 168         description => "List of additional CPU flags separated by ';'."
 169                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 170                      . " Currently supported flags: 'pcid', 'spec-ctrl'.",
 171         format_description => '+FLAG[;-FLAG...]',
 172         type => 'string',
 173         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 174         optional => 1,
 175     },
 176 };
 177
 178 my $watchdog_fmt = {
 179     model => {
 180         default_key => 1,
 181         type => 'string',
 182         enum => [qw(i6300esb ib700)],
 183         description => "Watchdog type to emulate.",
 184         default => 'i6300esb',
 185         optional => 1,
 186     },
 187     action => {
 188         type => 'string',
 189         enum => [qw(reset shutdown poweroff pause debug none)],
 190         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 191         optional => 1,
 192     },
 193 };
 194 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 195
 196 my $confdesc = {
 197     onboot => {
 198         optional => 1,
 199         type => 'boolean',
 200         description => "Specifies whether a VM will be started during system bootup.",
 201         default => 0,
 202     },
 203     autostart => {
 204         optional => 1,
 205         type => 'boolean',
 206         description => "Automatic restart after crash (currently ignored).",
 207         default => 0,
 208     },
 209     hotplug => {
 210         optional => 1,
 211         type => 'string', format => 'pve-hotplug-features',
 212         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 213         default => 'network,disk,usb',
 214     },
 215     reboot => {
 216         optional => 1,
 217         type => 'boolean',
 218         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 219         default => 1,
 220     },
 221     lock => {
 222         optional => 1,
 223         type => 'string',
 224         description => "Lock/unlock the VM.",
 225         enum => [qw(migrate backup snapshot rollback)],
 226     },
 227     cpulimit => {
 228         optional => 1,
 229         type => 'number',
 230         description => "Limit of CPU usage.",
 231         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 232         minimum => 0,
 233         maximum => 128,
 234         default => 0,
 235     },
 236     cpuunits => {
 237         optional => 1,
 238         type => 'integer',
 239         description => "CPU weight for a VM.",
 240         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 241         minimum => 2,
 242         maximum => 262144,
 243         default => 1024,
 244     },
 245     memory => {
 246         optional => 1,
 247         type => 'integer',
 248         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 249         minimum => 16,
 250         default => 512,
 251     },
 252     balloon => {
 253         optional => 1,
 254         type => 'integer',
 255         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 256         minimum => 0,
 257     },
 258     shares => {
 259         optional => 1,
 260         type => 'integer',
 261         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 262         minimum => 0,
 263         maximum => 50000,
 264         default => 1000,
 265     },
 266     keyboard => {
 267         optional => 1,
 268         type => 'string',
 269         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.".
 270                        "It should not be necessary to set it.",
 271         enum => PVE::Tools::kvmkeymaplist(),
 272         default => undef,
 273     },
 274     name => {
 275         optional => 1,
 276         type => 'string', format => 'dns-name',
 277         description => "Set a name for the VM. Only used on the configuration web interface.",
 278     },
 279     scsihw => {
 280         optional => 1,
 281         type => 'string',
 282         description => "SCSI controller model",
 283         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 284         default => 'lsi',
 285     },
 286     description => {
 287         optional => 1,
 288         type => 'string',
 289         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 290     },
 291     ostype => {
 292         optional => 1,
 293         type => 'string',
 294         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 295         description => "Specify guest operating system.",
 296         verbose_description => <<EODESC,
 297 Specify guest operating system. This is used to enable special
 298 optimization/features for specific operating systems:
 299
 300 [horizontal]
 301 other;; unspecified OS
 302 wxp;; Microsoft Windows XP
 303 w2k;; Microsoft Windows 2000
 304 w2k3;; Microsoft Windows 2003
 305 w2k8;; Microsoft Windows 2008
 306 wvista;; Microsoft Windows Vista
 307 win7;; Microsoft Windows 7
 308 win8;; Microsoft Windows 8/2012/2012r2
 309 win10;; Microsoft Windows 10/2016
 310 l24;; Linux 2.4 Kernel
 311 l26;; Linux 2.6/3.X Kernel
 312 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 313 EODESC
 314     },
 315     boot => {
 316         optional => 1,
 317         type => 'string',
 318         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 319         pattern => '[acdn]{1,4}',
 320         default => 'cdn',
 321     },
 322     bootdisk => {
 323         optional => 1,
 324         type => 'string', format => 'pve-qm-bootdisk',
 325         description => "Enable booting from specified disk.",
 326         pattern => '(ide|sata|scsi|virtio)\d+',
 327     },
 328     smp => {
 329         optional => 1,
 330         type => 'integer',
 331         description => "The number of CPUs. Please use option -sockets instead.",
 332         minimum => 1,
 333         default => 1,
 334     },
 335     sockets => {
 336         optional => 1,
 337         type => 'integer',
 338         description => "The number of CPU sockets.",
 339         minimum => 1,
 340         default => 1,
 341     },
 342     cores => {
 343         optional => 1,
 344         type => 'integer',
 345         description => "The number of cores per socket.",
 346         minimum => 1,
 347         default => 1,
 348     },
 349     numa => {
 350         optional => 1,
 351         type => 'boolean',
 352         description => "Enable/disable NUMA.",
 353         default => 0,
 354     },
 355     hugepages => {
 356         optional => 1,
 357         type => 'string',
 358         description => "Enable/disable hugepages memory.",
 359         enum => [qw(any 2 1024)],
 360     },
 361     vcpus => {
 362         optional => 1,
 363         type => 'integer',
 364         description => "Number of hotplugged vcpus.",
 365         minimum => 1,
 366         default => 0,
 367     },
 368     acpi => {
 369         optional => 1,
 370         type => 'boolean',
 371         description => "Enable/disable ACPI.",
 372         default => 1,
 373     },
 374     agent => {
 375         optional => 1,
 376         type => 'boolean',
 377         description => "Enable/disable Qemu GuestAgent.",
 378         default => 0,
 379     },
 380     kvm => {
 381         optional => 1,
 382         type => 'boolean',
 383         description => "Enable/disable KVM hardware virtualization.",
 384         default => 1,
 385     },
 386     tdf => {
 387         optional => 1,
 388         type => 'boolean',
 389         description => "Enable/disable time drift fix.",
 390         default => 0,
 391     },
 392     localtime => {
 393         optional => 1,
 394         type => 'boolean',
 395         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 396     },
 397     freeze => {
 398         optional => 1,
 399         type => 'boolean',
 400         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 401     },
 402     vga => {
 403         optional => 1,
 404         type => 'string',
 405         description => "Select the VGA type.",
 406         verbose_description => "Select the VGA type. If you want to use high resolution" .
 407             " modes (>= 1280x1024x16) then you should use the options " .
 408             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 409             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 410             "display sever. For win* OS you can select how many independent " .
 411             "displays you want, Linux guests can add displays them self. " .
 412             "You can also run without any graphic card, using a serial device" .
 413             " as terminal.",
 414         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 415     },
 416     watchdog => {
 417         optional => 1,
 418         type => 'string', format => 'pve-qm-watchdog',
 419         description => "Create a virtual hardware watchdog device.",
 420         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 421             " (by a guest action), the watchdog must be periodically polled " .
 422             "by an agent inside the guest or else the watchdog will reset " .
 423             "the guest (or execute the respective action specified)",
 424     },
 425     startdate => {
 426         optional => 1,
 427         type => 'string',
 428         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 429         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 430         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 431         default => 'now',
 432     },
 433     startup =>  get_standard_option('pve-startup-order'),
 434     template => {
 435         optional => 1,
 436         type => 'boolean',
 437         description => "Enable/disable Template.",
 438         default => 0,
 439     },
 440     args => {
 441         optional => 1,
 442         type => 'string',
 443         description => "Arbitrary arguments passed to kvm.",
 444         verbose_description => <<EODESCR,
 445 Arbitrary arguments passed to kvm, for example:
 446
 447 args: -no-reboot -no-hpet
 448
 449 NOTE: this option is for experts only.
 450 EODESCR
 451     },
 452     tablet => {
 453         optional => 1,
 454         type => 'boolean',
 455         default => 1,
 456         description => "Enable/disable the USB tablet device.",
 457         verbose_description => "Enable/disable the USB tablet device. This device is " .
 458             "usually needed to allow absolute mouse positioning with VNC. " .
 459             "Else the mouse runs out of sync with normal VNC clients. " .
 460             "If you're running lots of console-only guests on one host, " .
 461             "you may consider disabling this to save some context switches. " .
 462             "This is turned off by default if you use spice (-vga=qxl).",
 463     },
 464     migrate_speed => {
 465         optional => 1,
 466         type => 'integer',
 467         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 468         minimum => 0,
 469         default => 0,
 470     },
 471     migrate_downtime => {
 472         optional => 1,
 473         type => 'number',
 474         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 475         minimum => 0,
 476         default => 0.1,
 477     },
 478     cdrom => {
 479         optional => 1,
 480         type => 'string', format => 'pve-qm-ide',
 481         typetext => '<volume>',
 482         description => "This is an alias for option -ide2",
 483     },
 484     cpu => {
 485         optional => 1,
 486         description => "Emulated CPU type.",
 487         type => 'string',
 488         format => $cpu_fmt,
 489     },
 490     parent => get_standard_option('pve-snapshot-name', {
 491         optional => 1,
 492         description => "Parent snapshot name. This is used internally, and should not be modified.",
 493     }),
 494     snaptime => {
 495         optional => 1,
 496         description => "Timestamp for snapshots.",
 497         type => 'integer',
 498         minimum => 0,
 499     },
 500     vmstate => {
 501         optional => 1,
 502         type => 'string', format => 'pve-volume-id',
 503         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 504     },
 505     vmstatestorage => get_standard_option('pve-storage-id', {
 506         description => "Default storage for VM state volumes/files.",
 507         optional => 1,
 508     }),
 509     machine => {
 510         description => "Specific the Qemu machine type.",
 511         type => 'string',
 512         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 513         maxLength => 40,
 514         optional => 1,
 515     },
 516     smbios1 => {
 517         description => "Specify SMBIOS type 1 fields.",
 518         type => 'string', format => 'pve-qm-smbios1',
 519         maxLength => 256,
 520         optional => 1,
 521     },
 522     protection => {
 523         optional => 1,
 524         type => 'boolean',
 525         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 526         default => 0,
 527     },
 528     bios => {
 529         optional => 1,
 530         type => 'string',
 531         enum => [ qw(seabios ovmf) ],
 532         description => "Select BIOS implementation.",
 533         default => 'seabios',
 534     },
 535 };
 536
 537 # what about other qemu settings ?
 538 #cpu => 'string',
 539 #machine => 'string',
 540 #fda => 'file',
 541 #fdb => 'file',
 542 #mtdblock => 'file',
 543 #sd => 'file',
 544 #pflash => 'file',
 545 #snapshot => 'bool',
 546 #bootp => 'file',
 547 ##tftp => 'dir',
 548 ##smb => 'dir',
 549 #kernel => 'file',
 550 #append => 'string',
 551 #initrd => 'file',
 552 ##soundhw => 'string',
 553
 554 while (my ($k, $v) = each %$confdesc) {
 555     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 556 }
 557
 558 my $MAX_IDE_DISKS = 4;
 559 my $MAX_SCSI_DISKS = 14;
 560 my $MAX_VIRTIO_DISKS = 16;
 561 my $MAX_SATA_DISKS = 6;
 562 my $MAX_USB_DEVICES = 5;
 563 my $MAX_NETS = 32;
 564 my $MAX_UNUSED_DISKS = 8;
 565 my $MAX_HOSTPCI_DEVICES = 4;
 566 my $MAX_SERIAL_PORTS = 4;
 567 my $MAX_PARALLEL_PORTS = 3;
 568 my $MAX_NUMA = 8;
 569
 570 my $numa_fmt = {
 571     cpus => {
 572         type => "string",
 573         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 574         description => "CPUs accessing this NUMA node.",
 575         format_description => "id[-id];...",
 576     },
 577     memory => {
 578         type => "number",
 579         description => "Amount of memory this NUMA node provides.",
 580         optional => 1,
 581     },
 582     hostnodes => {
 583         type => "string",
 584         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 585         description => "Host NUMA nodes to use.",
 586         format_description => "id[-id];...",
 587         optional => 1,
 588     },
 589     policy => {
 590         type => 'string',
 591         enum => [qw(preferred bind interleave)],
 592         description => "NUMA allocation policy.",
 593         optional => 1,
 594     },
 595 };
 596 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 597 my $numadesc = {
 598     optional => 1,
 599     type => 'string', format => $numa_fmt,
 600     description => "NUMA topology.",
 601 };
 602 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 603
 604 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 605     $confdesc->{"numa$i"} = $numadesc;
 606 }
 607
 608 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 609                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 610                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 611 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 612
 613 my $net_fmt_bridge_descr = <<__EOD__;
 614 Bridge to attach the network device to. The Proxmox VE standard bridge
 615 is called 'vmbr0'.
 616
 617 If you do not specify a bridge, we create a kvm user (NATed) network
 618 device, which provides DHCP and DNS services. The following addresses
 619 are used:
 620
 621  10.0.2.2   Gateway
 622  10.0.2.3   DNS Server
 623  10.0.2.4   SMB Server
 624
 625 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 626 __EOD__
 627
 628 my $net_fmt = {
 629     macaddr => {
 630         type => 'string',
 631         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 632         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 633         format_description => "XX:XX:XX:XX:XX:XX",
 634         optional => 1,
 635     },
 636     model => {
 637         type => 'string',
 638         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 639         enum => $nic_model_list,
 640         default_key => 1,
 641     },
 642     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 643     bridge => {
 644         type => 'string',
 645         description => $net_fmt_bridge_descr,
 646         format_description => 'bridge',
 647         optional => 1,
 648     },
 649     queues => {
 650         type => 'integer',
 651         minimum => 0, maximum => 16,
 652         description => 'Number of packet queues to be used on the device.',
 653         optional => 1,
 654     },
 655     rate => {
 656         type => 'number',
 657         minimum => 0,
 658         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 659         optional => 1,
 660     },
 661     tag => {
 662         type => 'integer',
 663         minimum => 1, maximum => 4094,
 664         description => 'VLAN tag to apply to packets on this interface.',
 665         optional => 1,
 666     },
 667     trunks => {
 668         type => 'string',
 669         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 670         description => 'VLAN trunks to pass through this interface.',
 671         format_description => 'vlanid[;vlanid...]',
 672         optional => 1,
 673     },
 674     firewall => {
 675         type => 'boolean',
 676         description => 'Whether this interface should be protected by the firewall.',
 677         optional => 1,
 678     },
 679     link_down => {
 680         type => 'boolean',
 681         description => 'Whether this interface should be disconnected (like pulling the plug).',
 682         optional => 1,
 683     },
 684 };
 685
 686 my $netdesc = {
 687     optional => 1,
 688     type => 'string', format => $net_fmt,
 689     description => "Specify network devices.",
 690 };
 691
 692 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 693
 694 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 695     $confdesc->{"net$i"} = $netdesc;
 696 }
 697
 698 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 699 sub verify_volume_id_or_qm_path {
 700     my ($volid, $noerr) = @_;
 701
 702     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 703         return $volid;
 704     }
 705
 706     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 707     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 708     if ($@) {
 709         return undef if $noerr;
 710         die $@;
 711     }
 712     return $volid;
 713 }
 714
 715 my $drivename_hash;
 716
 717 my %drivedesc_base = (
 718     volume => { alias => 'file' },
 719     file => {
 720         type => 'string',
 721         format => 'pve-volume-id-or-qm-path',
 722         default_key => 1,
 723         format_description => 'volume',
 724         description => "The drive's backing volume.",
 725     },
 726     media => {
 727         type => 'string',
 728         enum => [qw(cdrom disk)],
 729         description => "The drive's media type.",
 730         default => 'disk',
 731         optional => 1
 732     },
 733     cyls => {
 734         type => 'integer',
 735         description => "Force the drive's physical geometry to have a specific cylinder count.",
 736         optional => 1
 737     },
 738     heads => {
 739         type => 'integer',
 740         description => "Force the drive's physical geometry to have a specific head count.",
 741         optional => 1
 742     },
 743     secs => {
 744         type => 'integer',
 745         description => "Force the drive's physical geometry to have a specific sector count.",
 746         optional => 1
 747     },
 748     trans => {
 749         type => 'string',
 750         enum => [qw(none lba auto)],
 751         description => "Force disk geometry bios translation mode.",
 752         optional => 1,
 753     },
 754     snapshot => {
 755         type => 'boolean',
 756         description => "Controls qemu's snapshot mode feature."
 757             . " If activated, changes made to the disk are temporary and will"
 758             . " be discarded when the VM is shutdown.",
 759         optional => 1,
 760     },
 761     cache => {
 762         type => 'string',
 763         enum => [qw(none writethrough writeback unsafe directsync)],
 764         description => "The drive's cache mode",
 765         optional => 1,
 766     },
 767     format => get_standard_option('pve-qm-image-format'),
 768     size => {
 769         type => 'string',
 770         format => 'disk-size',
 771         format_description => 'DiskSize',
 772         description => "Disk size. This is purely informational and has no effect.",
 773         optional => 1,
 774     },
 775     backup => {
 776         type => 'boolean',
 777         description => "Whether the drive should be included when making backups.",
 778         optional => 1,
 779     },
 780     replicate => {
 781         type => 'boolean',
 782         description => 'Whether the drive should considered for replication jobs.',
 783         optional => 1,
 784         default => 1,
 785     },
 786     rerror => {
 787         type => 'string',
 788         enum => [qw(ignore report stop)],
 789         description => 'Read error action.',
 790         optional => 1,
 791     },
 792     werror => {
 793         type => 'string',
 794         enum => [qw(enospc ignore report stop)],
 795         description => 'Write error action.',
 796         optional => 1,
 797     },
 798     aio => {
 799         type => 'string',
 800         enum => [qw(native threads)],
 801         description => 'AIO type to use.',
 802         optional => 1,
 803     },
 804     discard => {
 805         type => 'string',
 806         enum => [qw(ignore on)],
 807         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 808         optional => 1,
 809     },
 810     detect_zeroes => {
 811         type => 'boolean',
 812         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 813         optional => 1,
 814     },
 815     serial => {
 816         type => 'string',
 817         format => 'urlencoded',
 818         format_description => 'serial',
 819         maxLength => 20*3, # *3 since it's %xx url enoded
 820         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 821         optional => 1,
 822     },
 823     shared => {
 824         type => 'boolean',
 825         description => 'Mark this locally-managed volume as available on all nodes',
 826         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 827         optional => 1,
 828         default => 0,
 829     }
 830 );
 831
 832 my %iothread_fmt = ( iothread => {
 833         type => 'boolean',
 834         description => "Whether to use iothreads for this drive",
 835         optional => 1,
 836 });
 837
 838 my %model_fmt = (
 839     model => {
 840         type => 'string',
 841         format => 'urlencoded',
 842         format_description => 'model',
 843         maxLength => 40*3, # *3 since it's %xx url enoded
 844         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 845         optional => 1,
 846     },
 847 );
 848
 849 my %queues_fmt = (
 850     queues => {
 851         type => 'integer',
 852         description => "Number of queues.",
 853         minimum => 2,
 854         optional => 1
 855     }
 856 );
 857
 858 my %scsiblock_fmt = (
 859     scsiblock => {
 860         type => 'boolean',
 861         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 862         optional => 1,
 863         default => 0,
 864     },
 865 );
 866
 867 my $add_throttle_desc = sub {
 868     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 869     my $d = {
 870         type => $type,
 871         format_description => $unit,
 872         description => "Maximum $what in $longunit.",
 873         optional => 1,
 874     };
 875     $d->{minimum} = $minimum if defined($minimum);
 876     $drivedesc_base{$key} = $d;
 877 };
 878 # throughput: (leaky bucket)
 879 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 880 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 881 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 882 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 883 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 884 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 885 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 886 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 887 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 888
 889 # pools: (pool of IO before throttling starts taking effect)
 890 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 891 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 892 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 893 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 894 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 895 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 896
 897 # burst lengths
 898 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 899 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 900 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 901 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 902 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 903 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 904
 905 # legacy support
 906 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
 907 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
 908 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
 909 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
 910
 911 my $ide_fmt = {
 912     %drivedesc_base,
 913     %model_fmt,
 914 };
 915 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 916
 917 my $idedesc = {
 918     optional => 1,
 919     type => 'string', format => $ide_fmt,
 920     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 921 };
 922 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 923
 924 my $scsi_fmt = {
 925     %drivedesc_base,
 926     %iothread_fmt,
 927     %queues_fmt,
 928     %scsiblock_fmt,
 929 };
 930 my $scsidesc = {
 931     optional => 1,
 932     type => 'string', format => $scsi_fmt,
 933     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 934 };
 935 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 936
 937 my $sata_fmt = {
 938     %drivedesc_base,
 939 };
 940 my $satadesc = {
 941     optional => 1,
 942     type => 'string', format => $sata_fmt,
 943     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 944 };
 945 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 946
 947 my $virtio_fmt = {
 948     %drivedesc_base,
 949     %iothread_fmt,
 950 };
 951 my $virtiodesc = {
 952     optional => 1,
 953     type => 'string', format => $virtio_fmt,
 954     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 955 };
 956 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 957
 958 my $alldrive_fmt = {
 959     %drivedesc_base,
 960     %iothread_fmt,
 961     %model_fmt,
 962     %queues_fmt,
 963     %scsiblock_fmt,
 964 };
 965
 966 my $efidisk_fmt = {
 967     volume => { alias => 'file' },
 968     file => {
 969         type => 'string',
 970         format => 'pve-volume-id-or-qm-path',
 971         default_key => 1,
 972         format_description => 'volume',
 973         description => "The drive's backing volume.",
 974     },
 975     format => get_standard_option('pve-qm-image-format'),
 976     size => {
 977         type => 'string',
 978         format => 'disk-size',
 979         format_description => 'DiskSize',
 980         description => "Disk size. This is purely informational and has no effect.",
 981         optional => 1,
 982     },
 983 };
 984
 985 my $efidisk_desc = {
 986     optional => 1,
 987     type => 'string', format => $efidisk_fmt,
 988     description => "Configure a Disk for storing EFI vars",
 989 };
 990
 991 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 992
 993 my $usb_fmt = {
 994     host => {
 995         default_key => 1,
 996         type => 'string', format => 'pve-qm-usb-device',
 997         format_description => 'HOSTUSBDEVICE|spice',
 998         description => <<EODESCR,
 999 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1000
1001  'bus-port(.port)*' (decimal numbers) or
1002  'vendor_id:product_id' (hexadeciaml numbers) or
1003  'spice'
1004
1005 You can use the 'lsusb -t' command to list existing usb devices.
1006
1007 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1008
1009 The value 'spice' can be used to add a usb redirection devices for spice.
1010 EODESCR
1011     },
1012     usb3 => {
1013         optional => 1,
1014         type => 'boolean',
1015         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1016         default => 0,
1017     },
1018 };
1019
1020 my $usbdesc = {
1021     optional => 1,
1022     type => 'string', format => $usb_fmt,
1023     description => "Configure an USB device (n is 0 to 4).",
1024 };
1025 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1026
1027 # NOTE: the match-groups of this regex are used in parse_hostpci
1028 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1029 my $hostpci_fmt = {
1030     host => {
1031         default_key => 1,
1032         type => 'string',
1033         pattern => qr/$PCIRE(;$PCIRE)*/,
1034         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1035         description => <<EODESCR,
1036 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1037 of PCI virtual functions of the host. HOSTPCIID syntax is:
1038
1039 'bus:dev.func' (hexadecimal numbers)
1040
1041 You can us the 'lspci' command to list existing PCI devices.
1042 EODESCR
1043     },
1044     rombar => {
1045         type => 'boolean',
1046         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1047         optional => 1,
1048         default => 1,
1049     },
1050     romfile => {
1051         type => 'string',
1052         pattern => '[^,;]+',
1053         format_description => 'string',
1054         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1055         optional => 1,
1056     },
1057     pcie => {
1058         type => 'boolean',
1059         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1060         optional => 1,
1061         default => 0,
1062     },
1063     'x-vga' => {
1064         type => 'boolean',
1065         description =>  "Enable vfio-vga device support.",
1066         optional => 1,
1067         default => 0,
1068     },
1069 };
1070 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1071
1072 my $hostpcidesc = {
1073         optional => 1,
1074         type => 'string', format => 'pve-qm-hostpci',
1075         description => "Map host PCI devices into guest.",
1076         verbose_description =>  <<EODESCR,
1077 Map host PCI devices into guest.
1078
1079 NOTE: This option allows direct access to host hardware. So it is no longer
1080 possible to migrate such machines - use with special care.
1081
1082 CAUTION: Experimental! User reported problems with this option.
1083 EODESCR
1084 };
1085 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1086
1087 my $serialdesc = {
1088         optional => 1,
1089         type => 'string',
1090         pattern => '(/dev/.+|socket)',
1091         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1092         verbose_description =>  <<EODESCR,
1093 Create a serial device inside the VM (n is 0 to 3), and pass through a
1094 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1095 host side (use 'qm terminal' to open a terminal connection).
1096
1097 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1098
1099 CAUTION: Experimental! User reported problems with this option.
1100 EODESCR
1101 };
1102
1103 my $paralleldesc= {
1104         optional => 1,
1105         type => 'string',
1106         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1107         description =>  "Map host parallel devices (n is 0 to 2).",
1108         verbose_description =>  <<EODESCR,
1109 Map host parallel devices (n is 0 to 2).
1110
1111 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1112
1113 CAUTION: Experimental! User reported problems with this option.
1114 EODESCR
1115 };
1116
1117 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1118     $confdesc->{"parallel$i"} = $paralleldesc;
1119 }
1120
1121 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1122     $confdesc->{"serial$i"} = $serialdesc;
1123 }
1124
1125 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1126     $confdesc->{"hostpci$i"} = $hostpcidesc;
1127 }
1128
1129 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1130     $drivename_hash->{"ide$i"} = 1;
1131     $confdesc->{"ide$i"} = $idedesc;
1132 }
1133
1134 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1135     $drivename_hash->{"sata$i"} = 1;
1136     $confdesc->{"sata$i"} = $satadesc;
1137 }
1138
1139 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1140     $drivename_hash->{"scsi$i"} = 1;
1141     $confdesc->{"scsi$i"} = $scsidesc ;
1142 }
1143
1144 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1145     $drivename_hash->{"virtio$i"} = 1;
1146     $confdesc->{"virtio$i"} = $virtiodesc;
1147 }
1148
1149 $drivename_hash->{efidisk0} = 1;
1150 $confdesc->{efidisk0} = $efidisk_desc;
1151
1152 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1153     $confdesc->{"usb$i"} = $usbdesc;
1154 }
1155
1156 my $unuseddesc = {
1157     optional => 1,
1158     type => 'string', format => 'pve-volume-id',
1159     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1160 };
1161
1162 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1163     $confdesc->{"unused$i"} = $unuseddesc;
1164 }
1165
1166 my $kvm_api_version = 0;
1167
1168 sub kvm_version {
1169
1170     return $kvm_api_version if $kvm_api_version;
1171
1172     my $fh = IO::File->new("</dev/kvm") ||
1173         return 0;
1174
1175     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1176         $kvm_api_version = $v;
1177     }
1178
1179     $fh->close();
1180
1181     return  $kvm_api_version;
1182 }
1183
1184 my $kvm_user_version;
1185
1186 sub kvm_user_version {
1187
1188     return $kvm_user_version if $kvm_user_version;
1189
1190     $kvm_user_version = 'unknown';
1191
1192     my $code = sub {
1193         my $line = shift;
1194         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1195             $kvm_user_version = $2;
1196         }
1197     };
1198
1199     eval { run_command("kvm -version", outfunc => $code); };
1200     warn $@ if $@;
1201
1202     return $kvm_user_version;
1203
1204 }
1205
1206 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1207
1208 sub valid_drive_names {
1209     # order is important - used to autoselect boot disk
1210     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1211             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1212             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1213             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1214             'efidisk0');
1215 }
1216
1217 sub is_valid_drivename {
1218     my $dev = shift;
1219
1220     return defined($drivename_hash->{$dev});
1221 }
1222
1223 sub option_exists {
1224     my $key = shift;
1225     return defined($confdesc->{$key});
1226 }
1227
1228 sub nic_models {
1229     return $nic_model_list;
1230 }
1231
1232 sub os_list_description {
1233
1234     return {
1235         other => 'Other',
1236         wxp => 'Windows XP',
1237         w2k => 'Windows 2000',
1238         w2k3 =>, 'Windows 2003',
1239         w2k8 => 'Windows 2008',
1240         wvista => 'Windows Vista',
1241         win7 => 'Windows 7',
1242         win8 => 'Windows 8/2012',
1243         win10 => 'Windows 10/2016',
1244         l24 => 'Linux 2.4',
1245         l26 => 'Linux 2.6',
1246     };
1247 }
1248
1249 my $cdrom_path;
1250
1251 sub get_cdrom_path {
1252
1253     return  $cdrom_path if $cdrom_path;
1254
1255     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1256     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1257     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1258 }
1259
1260 sub get_iso_path {
1261     my ($storecfg, $vmid, $cdrom) = @_;
1262
1263     if ($cdrom eq 'cdrom') {
1264         return get_cdrom_path();
1265     } elsif ($cdrom eq 'none') {
1266         return '';
1267     } elsif ($cdrom =~ m|^/|) {
1268         return $cdrom;
1269     } else {
1270         return PVE::Storage::path($storecfg, $cdrom);
1271     }
1272 }
1273
1274 # try to convert old style file names to volume IDs
1275 sub filename_to_volume_id {
1276     my ($vmid, $file, $media) = @_;
1277
1278     if (!($file eq 'none' || $file eq 'cdrom' ||
1279           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1280
1281         return undef if $file =~ m|/|;
1282
1283         if ($media && $media eq 'cdrom') {
1284             $file = "local:iso/$file";
1285         } else {
1286             $file = "local:$vmid/$file";
1287         }
1288     }
1289
1290     return $file;
1291 }
1292
1293 sub verify_media_type {
1294     my ($opt, $vtype, $media) = @_;
1295
1296     return if !$media;
1297
1298     my $etype;
1299     if ($media eq 'disk') {
1300         $etype = 'images';
1301     } elsif ($media eq 'cdrom') {
1302         $etype = 'iso';
1303     } else {
1304         die "internal error";
1305     }
1306
1307     return if ($vtype eq $etype);
1308
1309     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1310 }
1311
1312 sub cleanup_drive_path {
1313     my ($opt, $storecfg, $drive) = @_;
1314
1315     # try to convert filesystem paths to volume IDs
1316
1317     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1318         ($drive->{file} !~ m|^/dev/.+|) &&
1319         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1320         ($drive->{file} !~ m/^\d+$/)) {
1321         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1322         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1323         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1324         verify_media_type($opt, $vtype, $drive->{media});
1325         $drive->{file} = $volid;
1326     }
1327
1328     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1329 }
1330
1331 sub parse_hotplug_features {
1332     my ($data) = @_;
1333
1334     my $res = {};
1335
1336     return $res if $data eq '0';
1337
1338     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1339
1340     foreach my $feature (PVE::Tools::split_list($data)) {
1341         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1342             $res->{$1} = 1;
1343         } else {
1344             die "invalid hotplug feature '$feature'\n";
1345         }
1346     }
1347     return $res;
1348 }
1349
1350 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1351 sub pve_verify_hotplug_features {
1352     my ($value, $noerr) = @_;
1353
1354     return $value if parse_hotplug_features($value);
1355
1356     return undef if $noerr;
1357
1358     die "unable to parse hotplug option\n";
1359 }
1360
1361 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1362 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1363 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1364 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1365 #        [,iothread=on][,serial=serial][,model=model]
1366
1367 sub parse_drive {
1368     my ($key, $data) = @_;
1369
1370     my ($interface, $index);
1371
1372     if ($key =~ m/^([^\d]+)(\d+)$/) {
1373         $interface = $1;
1374         $index = $2;
1375     } else {
1376         return undef;
1377     }
1378
1379     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1380                                      : $confdesc->{$key}->{format};
1381     if (!$desc) {
1382         warn "invalid drive key: $key\n";
1383         return undef;
1384     }
1385     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1386     return undef if !$res;
1387     $res->{interface} = $interface;
1388     $res->{index} = $index;
1389
1390     my $error = 0;
1391     foreach my $opt (qw(bps bps_rd bps_wr)) {
1392         if (my $bps = defined(delete $res->{$opt})) {
1393             if (defined($res->{"m$opt"})) {
1394                 warn "both $opt and m$opt specified\n";
1395                 ++$error;
1396                 next;
1397             }
1398             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1399         }
1400     }
1401
1402     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1403     for my $requirement (
1404         [mbps_max => 'mbps'],
1405         [mbps_rd_max => 'mbps_rd'],
1406         [mbps_wr_max => 'mbps_wr'],
1407         [miops_max => 'miops'],
1408         [miops_rd_max => 'miops_rd'],
1409         [miops_wr_max => 'miops_wr'],
1410         [bps_max_length => 'mbps_max'],
1411         [bps_rd_max_length => 'mbps_rd_max'],
1412         [bps_wr_max_length => 'mbps_wr_max'],
1413         [iops_max_length => 'iops_max'],
1414         [iops_rd_max_length => 'iops_rd_max'],
1415         [iops_wr_max_length => 'iops_wr_max']) {
1416         my ($option, $requires) = @$requirement;
1417         if ($res->{$option} && !$res->{$requires}) {
1418             warn "$option requires $requires\n";
1419             ++$error;
1420         }
1421     }
1422
1423     return undef if $error;
1424
1425     return undef if $res->{mbps_rd} && $res->{mbps};
1426     return undef if $res->{mbps_wr} && $res->{mbps};
1427     return undef if $res->{iops_rd} && $res->{iops};
1428     return undef if $res->{iops_wr} && $res->{iops};
1429
1430     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1431         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1432         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1433         return undef if $res->{interface} eq 'virtio';
1434     }
1435
1436     if (my $size = $res->{size}) {
1437         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1438     }
1439
1440     return $res;
1441 }
1442
1443 sub print_drive {
1444     my ($vmid, $drive) = @_;
1445     my $data = { %$drive };
1446     delete $data->{$_} for qw(index interface);
1447     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1448 }
1449
1450 sub scsi_inquiry {
1451     my($fh, $noerr) = @_;
1452
1453     my $SG_IO = 0x2285;
1454     my $SG_GET_VERSION_NUM = 0x2282;
1455
1456     my $versionbuf = "\x00" x 8;
1457     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1458     if (!$ret) {
1459         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1460         return undef;
1461     }
1462     my $version = unpack("I", $versionbuf);
1463     if ($version < 30000) {
1464         die "scsi generic interface too old\n"  if !$noerr;
1465         return undef;
1466     }
1467
1468     my $buf = "\x00" x 36;
1469     my $sensebuf = "\x00" x 8;
1470     my $cmd = pack("C x3 C x1", 0x12, 36);
1471
1472     # see /usr/include/scsi/sg.h
1473     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1474
1475     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1476                       length($sensebuf), 0, length($buf), $buf,
1477                       $cmd, $sensebuf, 6000);
1478
1479     $ret = ioctl($fh, $SG_IO, $packet);
1480     if (!$ret) {
1481         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1482         return undef;
1483     }
1484
1485     my @res = unpack($sg_io_hdr_t, $packet);
1486     if ($res[17] || $res[18]) {
1487         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1488         return undef;
1489     }
1490
1491     my $res = {};
1492     (my $byte0, my $byte1, $res->{vendor},
1493      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1494
1495     $res->{removable} = $byte1 & 128 ? 1 : 0;
1496     $res->{type} = $byte0 & 31;
1497
1498     return $res;
1499 }
1500
1501 sub path_is_scsi {
1502     my ($path) = @_;
1503
1504     my $fh = IO::File->new("+<$path") || return undef;
1505     my $res = scsi_inquiry($fh, 1);
1506     close($fh);
1507
1508     return $res;
1509 }
1510
1511 sub machine_type_is_q35 {
1512     my ($conf) = @_;
1513
1514     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1515 }
1516
1517 sub print_tabletdevice_full {
1518     my ($conf) = @_;
1519
1520     my $q35 = machine_type_is_q35($conf);
1521
1522     # we use uhci for old VMs because tablet driver was buggy in older qemu
1523     my $usbbus = $q35 ? "ehci" : "uhci";
1524
1525     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1526 }
1527
1528 sub print_drivedevice_full {
1529     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1530
1531     my $device = '';
1532     my $maxdev = 0;
1533
1534     if ($drive->{interface} eq 'virtio') {
1535         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1536         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1537         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1538     } elsif ($drive->{interface} eq 'scsi') {
1539
1540         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1541         my $unit = $drive->{index} % $maxdev;
1542         my $devicetype = 'hd';
1543         my $path = '';
1544         if (drive_is_cdrom($drive)) {
1545             $devicetype = 'cd';
1546         } else {
1547             if ($drive->{file} =~ m|^/|) {
1548                 $path = $drive->{file};
1549                 if (my $info = path_is_scsi($path)) {
1550                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1551                         $devicetype = 'block';
1552                     } elsif ($info->{type} == 1) { # tape
1553                         $devicetype = 'generic';
1554                     }
1555                 }
1556             } else {
1557                  $path = PVE::Storage::path($storecfg, $drive->{file});
1558             }
1559
1560             if($path =~ m/^iscsi\:\/\//){
1561                 $devicetype = 'generic';
1562             }
1563         }
1564
1565         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1566             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1567         } else {
1568             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1569         }
1570
1571     } elsif ($drive->{interface} eq 'ide'){
1572         $maxdev = 2;
1573         my $controller = int($drive->{index} / $maxdev);
1574         my $unit = $drive->{index} % $maxdev;
1575         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1576
1577         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1578         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1579             $model = URI::Escape::uri_unescape($model);
1580             $device .= ",model=$model";
1581         }
1582     } elsif ($drive->{interface} eq 'sata'){
1583         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1584         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1585         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1586     } elsif ($drive->{interface} eq 'usb') {
1587         die "implement me";
1588         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1589     } else {
1590         die "unsupported interface type";
1591     }
1592
1593     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1594
1595     return $device;
1596 }
1597
1598 sub get_initiator_name {
1599     my $initiator;
1600
1601     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1602     while (defined(my $line = <$fh>)) {
1603         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1604         $initiator = $1;
1605         last;
1606     }
1607     $fh->close();
1608
1609     return $initiator;
1610 }
1611
1612 sub print_drive_full {
1613     my ($storecfg, $vmid, $drive) = @_;
1614
1615     my $path;
1616     my $volid = $drive->{file};
1617     my $format;
1618
1619     if (drive_is_cdrom($drive)) {
1620         $path = get_iso_path($storecfg, $vmid, $volid);
1621     } else {
1622         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1623         if ($storeid) {
1624             $path = PVE::Storage::path($storecfg, $volid);
1625             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1626             $format = qemu_img_format($scfg, $volname);
1627         } else {
1628             $path = $volid;
1629             $format = "raw";
1630         }
1631    }
1632
1633     my $opts = '';
1634     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1635     foreach my $o (@qemu_drive_options) {
1636         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1637     }
1638
1639     # snapshot only accepts on|off
1640     if (defined($drive->{snapshot})) {
1641         my $v = $drive->{snapshot} ? 'on' : 'off';
1642         $opts .= ",snapshot=$v";
1643     }
1644
1645     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1646         my ($dir, $qmpname) = @$type;
1647         if (my $v = $drive->{"mbps$dir"}) {
1648             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1649         }
1650         if (my $v = $drive->{"mbps${dir}_max"}) {
1651             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1652         }
1653         if (my $v = $drive->{"bps${dir}_max_length"}) {
1654             $opts .= ",throttling.bps$qmpname-max-length=$v";
1655         }
1656         if (my $v = $drive->{"iops${dir}"}) {
1657             $opts .= ",throttling.iops$qmpname=$v";
1658         }
1659         if (my $v = $drive->{"iops${dir}_max"}) {
1660             $opts .= ",throttling.iops$qmpname-max=$v";
1661         }
1662         if (my $v = $drive->{"iops${dir}_max_length"}) {
1663             $opts .= ",throttling.iops$qmpname-max-length=$v";
1664         }
1665     }
1666
1667     if (my $serial = $drive->{serial}) {
1668         $serial = URI::Escape::uri_unescape($serial);
1669         $opts .= ",serial=$serial";
1670     }
1671
1672     $opts .= ",format=$format" if $format && !$drive->{format};
1673
1674     my $cache_direct = 0;
1675
1676     if (my $cache = $drive->{cache}) {
1677         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1678     } elsif (!drive_is_cdrom($drive)) {
1679         $opts .= ",cache=none";
1680         $cache_direct = 1;
1681     }
1682
1683     # aio native works only with O_DIRECT
1684     if (!$drive->{aio}) {
1685         if($cache_direct) {
1686             $opts .= ",aio=native";
1687         } else {
1688             $opts .= ",aio=threads";
1689         }
1690     }
1691
1692     if (!drive_is_cdrom($drive)) {
1693         my $detectzeroes;
1694         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1695             $detectzeroes = 'off';
1696         } elsif ($drive->{discard}) {
1697             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1698         } else {
1699             # This used to be our default with discard not being specified:
1700             $detectzeroes = 'on';
1701         }
1702         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1703     }
1704
1705     my $pathinfo = $path ? "file=$path," : '';
1706
1707     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1708 }
1709
1710 sub print_netdevice_full {
1711     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1712
1713     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1714
1715     my $device = $net->{model};
1716     if ($net->{model} eq 'virtio') {
1717          $device = 'virtio-net-pci';
1718      };
1719
1720     my $pciaddr = print_pci_addr("$netid", $bridges);
1721     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1722     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1723         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1724         my $vectors = $net->{queues} * 2 + 2;
1725         $tmpstr .= ",vectors=$vectors,mq=on";
1726     }
1727     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1728
1729     if ($use_old_bios_files) {
1730         my $romfile;
1731         if ($device eq 'virtio-net-pci') {
1732             $romfile = 'pxe-virtio.rom';
1733         } elsif ($device eq 'e1000') {
1734             $romfile = 'pxe-e1000.rom';
1735         } elsif ($device eq 'ne2k') {
1736             $romfile = 'pxe-ne2k_pci.rom';
1737         } elsif ($device eq 'pcnet') {
1738             $romfile = 'pxe-pcnet.rom';
1739         } elsif ($device eq 'rtl8139') {
1740             $romfile = 'pxe-rtl8139.rom';
1741         }
1742         $tmpstr .= ",romfile=$romfile" if $romfile;
1743     }
1744
1745     return $tmpstr;
1746 }
1747
1748 sub print_netdev_full {
1749     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1750
1751     my $i = '';
1752     if ($netid =~ m/^net(\d+)$/) {
1753         $i = int($1);
1754     }
1755
1756     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1757
1758     my $ifname = "tap${vmid}i$i";
1759
1760     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1761     die "interface name '$ifname' is too long (max 15 character)\n"
1762         if length($ifname) >= 16;
1763
1764     my $vhostparam = '';
1765     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1766
1767     my $vmname = $conf->{name} || "vm$vmid";
1768
1769     my $netdev = "";
1770     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1771
1772     if ($net->{bridge}) {
1773         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1774     } else {
1775         $netdev = "type=user,id=$netid,hostname=$vmname";
1776     }
1777
1778     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1779
1780     return $netdev;
1781 }
1782
1783
1784 sub print_cpu_device {
1785     my ($conf, $id) = @_;
1786
1787     my $kvm = $conf->{kvm} // 1;
1788     my $cpu = $kvm ? "kvm64" : "qemu64";
1789     if (my $cputype = $conf->{cpu}) {
1790         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1791             or die "Cannot parse cpu description: $cputype\n";
1792         $cpu = $cpuconf->{cputype};
1793     }
1794
1795     my $cores = $conf->{cores} || 1;
1796
1797     my $current_core = ($id - 1) % $cores;
1798     my $current_socket = int(($id - 1 - $current_core)/$cores);
1799
1800     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1801 }
1802
1803 sub drive_is_cdrom {
1804     my ($drive) = @_;
1805
1806     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1807
1808 }
1809
1810 sub parse_number_sets {
1811     my ($set) = @_;
1812     my $res = [];
1813     foreach my $part (split(/;/, $set)) {
1814         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1815             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1816             push @$res, [ $1, $2 ];
1817         } else {
1818             die "invalid range: $part\n";
1819         }
1820     }
1821     return $res;
1822 }
1823
1824 sub parse_numa {
1825     my ($data) = @_;
1826
1827     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1828     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1829     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1830     return $res;
1831 }
1832
1833 sub parse_hostpci {
1834     my ($value) = @_;
1835
1836     return undef if !$value;
1837
1838     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1839
1840     my @idlist = split(/;/, $res->{host});
1841     delete $res->{host};
1842     foreach my $id (@idlist) {
1843         if ($id =~ /^$PCIRE$/) {
1844             if (defined($2)) {
1845                 push @{$res->{pciid}}, { id => $1, function => $2 };
1846             } else {
1847                 my $pcidevices = lspci($1);
1848                 $res->{pciid} = $pcidevices->{$1};
1849             }
1850         } else {
1851             # should have been caught by parse_property_string already
1852             die "failed to parse PCI id: $id\n";
1853         }
1854     }
1855     return $res;
1856 }
1857
1858 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1859 sub parse_net {
1860     my ($data) = @_;
1861
1862     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1863     if ($@) {
1864         warn $@;
1865         return undef;
1866     }
1867     if (!defined($res->{macaddr})) {
1868         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1869         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1870     }
1871     return $res;
1872 }
1873
1874 sub print_net {
1875     my $net = shift;
1876
1877     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1878 }
1879
1880 sub add_random_macs {
1881     my ($settings) = @_;
1882
1883     foreach my $opt (keys %$settings) {
1884         next if $opt !~ m/^net(\d+)$/;
1885         my $net = parse_net($settings->{$opt});
1886         next if !$net;
1887         $settings->{$opt} = print_net($net);
1888     }
1889 }
1890
1891 sub vm_is_volid_owner {
1892     my ($storecfg, $vmid, $volid) = @_;
1893
1894     if ($volid !~  m|^/|) {
1895         my ($path, $owner);
1896         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1897         if ($owner && ($owner == $vmid)) {
1898             return 1;
1899         }
1900     }
1901
1902     return undef;
1903 }
1904
1905 sub split_flagged_list {
1906     my $text = shift || '';
1907     $text =~ s/[,;]/ /g;
1908     $text =~ s/^\s+//;
1909     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1910 }
1911
1912 sub join_flagged_list {
1913     my ($how, $lst) = @_;
1914     join $how, map { $lst->{$_} . $_ } keys %$lst;
1915 }
1916
1917 sub vmconfig_delete_pending_option {
1918     my ($conf, $key, $force) = @_;
1919
1920     delete $conf->{pending}->{$key};
1921     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1922     $pending_delete_hash->{$key} = $force ? '!' : '';
1923     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1924 }
1925
1926 sub vmconfig_undelete_pending_option {
1927     my ($conf, $key) = @_;
1928
1929     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1930     delete $pending_delete_hash->{$key};
1931
1932     if (%$pending_delete_hash) {
1933         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1934     } else {
1935         delete $conf->{pending}->{delete};
1936     }
1937 }
1938
1939 sub vmconfig_register_unused_drive {
1940     my ($storecfg, $vmid, $conf, $drive) = @_;
1941
1942     if (!drive_is_cdrom($drive)) {
1943         my $volid = $drive->{file};
1944         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1945             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1946         }
1947     }
1948 }
1949
1950 sub vmconfig_cleanup_pending {
1951     my ($conf) = @_;
1952
1953     # remove pending changes when nothing changed
1954     my $changes;
1955     foreach my $opt (keys %{$conf->{pending}}) {
1956         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1957             $changes = 1;
1958             delete $conf->{pending}->{$opt};
1959         }
1960     }
1961
1962     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1963     my $pending_delete_hash = {};
1964     while (my ($opt, $force) = each %$current_delete_hash) {
1965         if (defined($conf->{$opt})) {
1966             $pending_delete_hash->{$opt} = $force;
1967         } else {
1968             $changes = 1;
1969         }
1970     }
1971
1972     if (%$pending_delete_hash) {
1973         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1974     } else {
1975         delete $conf->{pending}->{delete};
1976     }
1977
1978     return $changes;
1979 }
1980
1981 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1982 my $smbios1_fmt = {
1983     uuid => {
1984         type => 'string',
1985         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1986         format_description => 'UUID',
1987         description => "Set SMBIOS1 UUID.",
1988         optional => 1,
1989     },
1990     version => {
1991         type => 'string',
1992         pattern => '\S+',
1993         format_description => 'string',
1994         description => "Set SMBIOS1 version.",
1995         optional => 1,
1996     },
1997     serial => {
1998         type => 'string',
1999         pattern => '\S+',
2000         format_description => 'string',
2001         description => "Set SMBIOS1 serial number.",
2002         optional => 1,
2003     },
2004     manufacturer => {
2005         type => 'string',
2006         pattern => '\S+',
2007         format_description => 'string',
2008         description => "Set SMBIOS1 manufacturer.",
2009         optional => 1,
2010     },
2011     product => {
2012         type => 'string',
2013         pattern => '\S+',
2014         format_description => 'string',
2015         description => "Set SMBIOS1 product ID.",
2016         optional => 1,
2017     },
2018     sku => {
2019         type => 'string',
2020         pattern => '\S+',
2021         format_description => 'string',
2022         description => "Set SMBIOS1 SKU string.",
2023         optional => 1,
2024     },
2025     family => {
2026         type => 'string',
2027         pattern => '\S+',
2028         format_description => 'string',
2029         description => "Set SMBIOS1 family string.",
2030         optional => 1,
2031     },
2032 };
2033
2034 sub parse_smbios1 {
2035     my ($data) = @_;
2036
2037     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2038     warn $@ if $@;
2039     return $res;
2040 }
2041
2042 sub print_smbios1 {
2043     my ($smbios1) = @_;
2044     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2045 }
2046
2047 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2048
2049 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2050 sub verify_bootdisk {
2051     my ($value, $noerr) = @_;
2052
2053     return $value if is_valid_drivename($value);
2054
2055     return undef if $noerr;
2056
2057     die "invalid boot disk '$value'\n";
2058 }
2059
2060 sub parse_watchdog {
2061     my ($value) = @_;
2062
2063     return undef if !$value;
2064
2065     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2066     warn $@ if $@;
2067     return $res;
2068 }
2069
2070 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2071 sub verify_usb_device {
2072     my ($value, $noerr) = @_;
2073
2074     return $value if parse_usb_device($value);
2075
2076     return undef if $noerr;
2077
2078     die "unable to parse usb device\n";
2079 }
2080
2081 # add JSON properties for create and set function
2082 sub json_config_properties {
2083     my $prop = shift;
2084
2085     foreach my $opt (keys %$confdesc) {
2086         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2087         $prop->{$opt} = $confdesc->{$opt};
2088     }
2089
2090     return $prop;
2091 }
2092
2093 sub check_type {
2094     my ($key, $value) = @_;
2095
2096     die "unknown setting '$key'\n" if !$confdesc->{$key};
2097
2098     my $type = $confdesc->{$key}->{type};
2099
2100     if (!defined($value)) {
2101         die "got undefined value\n";
2102     }
2103
2104     if ($value =~ m/[\n\r]/) {
2105         die "property contains a line feed\n";
2106     }
2107
2108     if ($type eq 'boolean') {
2109         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2110         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2111         die "type check ('boolean') failed - got '$value'\n";
2112     } elsif ($type eq 'integer') {
2113         return int($1) if $value =~ m/^(\d+)$/;
2114         die "type check ('integer') failed - got '$value'\n";
2115     } elsif ($type eq 'number') {
2116         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2117         die "type check ('number') failed - got '$value'\n";
2118     } elsif ($type eq 'string') {
2119         if (my $fmt = $confdesc->{$key}->{format}) {
2120             PVE::JSONSchema::check_format($fmt, $value);
2121             return $value;
2122         }
2123         $value =~ s/^\"(.*)\"$/$1/;
2124         return $value;
2125     } else {
2126         die "internal error"
2127     }
2128 }
2129
2130 sub check_iommu_support{
2131     #fixme : need to check IOMMU support
2132     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2133
2134     my $iommu=1;
2135     return $iommu;
2136
2137 }
2138
2139 sub touch_config {
2140     my ($vmid) = @_;
2141
2142     my $conf = PVE::QemuConfig->config_file($vmid);
2143     utime undef, undef, $conf;
2144 }
2145
2146 sub destroy_vm {
2147     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2148
2149     my $conffile = PVE::QemuConfig->config_file($vmid);
2150
2151     my $conf = PVE::QemuConfig->load_config($vmid);
2152
2153     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2154
2155     if ($conf->{template}) {
2156         # check if any base image is still used by a linked clone
2157         foreach_drive($conf, sub {
2158                 my ($ds, $drive) = @_;
2159
2160                 return if drive_is_cdrom($drive);
2161
2162                 my $volid = $drive->{file};
2163
2164                 return if !$volid || $volid =~ m|^/|;
2165
2166                 die "base volume '$volid' is still in use by linked cloned\n"
2167                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2168
2169         });
2170     }
2171
2172     # only remove disks owned by this VM
2173     foreach_drive($conf, sub {
2174         my ($ds, $drive) = @_;
2175
2176         return if drive_is_cdrom($drive);
2177
2178         my $volid = $drive->{file};
2179
2180         return if !$volid || $volid =~ m|^/|;
2181
2182         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2183         return if !$path || !$owner || ($owner != $vmid);
2184
2185         eval {
2186             PVE::Storage::vdisk_free($storecfg, $volid);
2187         };
2188         warn "Could not remove disk '$volid', check manually: $@" if $@;
2189
2190     });
2191
2192     if ($keep_empty_config) {
2193         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2194     } else {
2195         unlink $conffile;
2196     }
2197
2198     # also remove unused disk
2199     eval {
2200         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2201
2202         eval {
2203             PVE::Storage::foreach_volid($dl, sub {
2204                 my ($volid, $sid, $volname, $d) = @_;
2205                 PVE::Storage::vdisk_free($storecfg, $volid);
2206             });
2207         };
2208         warn $@ if $@;
2209
2210     };
2211     warn $@ if $@;
2212 }
2213
2214 sub parse_vm_config {
2215     my ($filename, $raw) = @_;
2216
2217     return undef if !defined($raw);
2218
2219     my $res = {
2220         digest => Digest::SHA::sha1_hex($raw),
2221         snapshots => {},
2222         pending => {},
2223     };
2224
2225     $filename =~ m|/qemu-server/(\d+)\.conf$|
2226         || die "got strange filename '$filename'";
2227
2228     my $vmid = $1;
2229
2230     my $conf = $res;
2231     my $descr;
2232     my $section = '';
2233
2234     my @lines = split(/\n/, $raw);
2235     foreach my $line (@lines) {
2236         next if $line =~ m/^\s*$/;
2237
2238         if ($line =~ m/^\[PENDING\]\s*$/i) {
2239             $section = 'pending';
2240             if (defined($descr)) {
2241                 $descr =~ s/\s+$//;
2242                 $conf->{description} = $descr;
2243             }
2244             $descr = undef;
2245             $conf = $res->{$section} = {};
2246             next;
2247
2248         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2249             $section = $1;
2250             if (defined($descr)) {
2251                 $descr =~ s/\s+$//;
2252                 $conf->{description} = $descr;
2253             }
2254             $descr = undef;
2255             $conf = $res->{snapshots}->{$section} = {};
2256             next;
2257         }
2258
2259         if ($line =~ m/^\#(.*)\s*$/) {
2260             $descr = '' if !defined($descr);
2261             $descr .= PVE::Tools::decode_text($1) . "\n";
2262             next;
2263         }
2264
2265         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2266             $descr = '' if !defined($descr);
2267             $descr .= PVE::Tools::decode_text($2);
2268         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2269             $conf->{snapstate} = $1;
2270         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2271             my $key = $1;
2272             my $value = $2;
2273             $conf->{$key} = $value;
2274         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2275             my $value = $1;
2276             if ($section eq 'pending') {
2277                 $conf->{delete} = $value; # we parse this later
2278             } else {
2279                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2280             }
2281         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2282             my $key = $1;
2283             my $value = $2;
2284             eval { $value = check_type($key, $value); };
2285             if ($@) {
2286                 warn "vm $vmid - unable to parse value of '$key' - $@";
2287             } else {
2288                 $key = 'ide2' if $key eq 'cdrom';
2289                 my $fmt = $confdesc->{$key}->{format};
2290                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2291                     my $v = parse_drive($key, $value);
2292                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2293                         $v->{file} = $volid;
2294                         $value = print_drive($vmid, $v);
2295                     } else {
2296                         warn "vm $vmid - unable to parse value of '$key'\n";
2297                         next;
2298                     }
2299                 }
2300
2301                 $conf->{$key} = $value;
2302             }
2303         }
2304     }
2305
2306     if (defined($descr)) {
2307         $descr =~ s/\s+$//;
2308         $conf->{description} = $descr;
2309     }
2310     delete $res->{snapstate}; # just to be sure
2311
2312     return $res;
2313 }
2314
2315 sub write_vm_config {
2316     my ($filename, $conf) = @_;
2317
2318     delete $conf->{snapstate}; # just to be sure
2319
2320     if ($conf->{cdrom}) {
2321         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2322         $conf->{ide2} = $conf->{cdrom};
2323         delete $conf->{cdrom};
2324     }
2325
2326     # we do not use 'smp' any longer
2327     if ($conf->{sockets}) {
2328         delete $conf->{smp};
2329     } elsif ($conf->{smp}) {
2330         $conf->{sockets} = $conf->{smp};
2331         delete $conf->{cores};
2332         delete $conf->{smp};
2333     }
2334
2335     my $used_volids = {};
2336
2337     my $cleanup_config = sub {
2338         my ($cref, $pending, $snapname) = @_;
2339
2340         foreach my $key (keys %$cref) {
2341             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2342                 $key eq 'snapstate' || $key eq 'pending';
2343             my $value = $cref->{$key};
2344             if ($key eq 'delete') {
2345                 die "propertry 'delete' is only allowed in [PENDING]\n"
2346                     if !$pending;
2347                 # fixme: check syntax?
2348                 next;
2349             }
2350             eval { $value = check_type($key, $value); };
2351             die "unable to parse value of '$key' - $@" if $@;
2352
2353             $cref->{$key} = $value;
2354
2355             if (!$snapname && is_valid_drivename($key)) {
2356                 my $drive = parse_drive($key, $value);
2357                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2358             }
2359         }
2360     };
2361
2362     &$cleanup_config($conf);
2363
2364     &$cleanup_config($conf->{pending}, 1);
2365
2366     foreach my $snapname (keys %{$conf->{snapshots}}) {
2367         die "internal error" if $snapname eq 'pending';
2368         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2369     }
2370
2371     # remove 'unusedX' settings if we re-add a volume
2372     foreach my $key (keys %$conf) {
2373         my $value = $conf->{$key};
2374         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2375             delete $conf->{$key};
2376         }
2377     }
2378
2379     my $generate_raw_config = sub {
2380         my ($conf, $pending) = @_;
2381
2382         my $raw = '';
2383
2384         # add description as comment to top of file
2385         if (defined(my $descr = $conf->{description})) {
2386             if ($descr) {
2387                 foreach my $cl (split(/\n/, $descr)) {
2388                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2389                 }
2390             } else {
2391                 $raw .= "#\n" if $pending;
2392             }
2393         }
2394
2395         foreach my $key (sort keys %$conf) {
2396             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2397             $raw .= "$key: $conf->{$key}\n";
2398         }
2399         return $raw;
2400     };
2401
2402     my $raw = &$generate_raw_config($conf);
2403
2404     if (scalar(keys %{$conf->{pending}})){
2405         $raw .= "\n[PENDING]\n";
2406         $raw .= &$generate_raw_config($conf->{pending}, 1);
2407     }
2408
2409     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2410         $raw .= "\n[$snapname]\n";
2411         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2412     }
2413
2414     return $raw;
2415 }
2416
2417 sub load_defaults {
2418
2419     my $res = {};
2420
2421     # we use static defaults from our JSON schema configuration
2422     foreach my $key (keys %$confdesc) {
2423         if (defined(my $default = $confdesc->{$key}->{default})) {
2424             $res->{$key} = $default;
2425         }
2426     }
2427
2428     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2429     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2430
2431     return $res;
2432 }
2433
2434 sub config_list {
2435     my $vmlist = PVE::Cluster::get_vmlist();
2436     my $res = {};
2437     return $res if !$vmlist || !$vmlist->{ids};
2438     my $ids = $vmlist->{ids};
2439
2440     foreach my $vmid (keys %$ids) {
2441         my $d = $ids->{$vmid};
2442         next if !$d->{node} || $d->{node} ne $nodename;
2443         next if !$d->{type} || $d->{type} ne 'qemu';
2444         $res->{$vmid}->{exists} = 1;
2445     }
2446     return $res;
2447 }
2448
2449 # test if VM uses local resources (to prevent migration)
2450 sub check_local_resources {
2451     my ($conf, $noerr) = @_;
2452
2453     my $loc_res = 0;
2454
2455     $loc_res = 1 if $conf->{hostusb}; # old syntax
2456     $loc_res = 1 if $conf->{hostpci}; # old syntax
2457
2458     foreach my $k (keys %$conf) {
2459         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2460         # sockets are safe: they will recreated be on the target side post-migrate
2461         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2462         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2463     }
2464
2465     die "VM uses local resources\n" if $loc_res && !$noerr;
2466
2467     return $loc_res;
2468 }
2469
2470 # check if used storages are available on all nodes (use by migrate)
2471 sub check_storage_availability {
2472     my ($storecfg, $conf, $node) = @_;
2473
2474     foreach_drive($conf, sub {
2475         my ($ds, $drive) = @_;
2476
2477         my $volid = $drive->{file};
2478         return if !$volid;
2479
2480         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2481         return if !$sid;
2482
2483         # check if storage is available on both nodes
2484         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2485         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2486    });
2487 }
2488
2489 # list nodes where all VM images are available (used by has_feature API)
2490 sub shared_nodes {
2491     my ($conf, $storecfg) = @_;
2492
2493     my $nodelist = PVE::Cluster::get_nodelist();
2494     my $nodehash = { map { $_ => 1 } @$nodelist };
2495     my $nodename = PVE::INotify::nodename();
2496
2497     foreach_drive($conf, sub {
2498         my ($ds, $drive) = @_;
2499
2500         my $volid = $drive->{file};
2501         return if !$volid;
2502
2503         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2504         if ($storeid) {
2505             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2506             if ($scfg->{disable}) {
2507                 $nodehash = {};
2508             } elsif (my $avail = $scfg->{nodes}) {
2509                 foreach my $node (keys %$nodehash) {
2510                     delete $nodehash->{$node} if !$avail->{$node};
2511                 }
2512             } elsif (!$scfg->{shared}) {
2513                 foreach my $node (keys %$nodehash) {
2514                     delete $nodehash->{$node} if $node ne $nodename
2515                 }
2516             }
2517         }
2518     });
2519
2520     return $nodehash
2521 }
2522
2523 sub check_cmdline {
2524     my ($pidfile, $pid) = @_;
2525
2526     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2527     if (defined($fh)) {
2528         my $line = <$fh>;
2529         $fh->close;
2530         return undef if !$line;
2531         my @param = split(/\0/, $line);
2532
2533         my $cmd = $param[0];
2534         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2535
2536         for (my $i = 0; $i < scalar (@param); $i++) {
2537             my $p = $param[$i];
2538             next if !$p;
2539             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2540                 my $p = $param[$i+1];
2541                 return 1 if $p && ($p eq $pidfile);
2542                 return undef;
2543             }
2544         }
2545     }
2546     return undef;
2547 }
2548
2549 sub check_running {
2550     my ($vmid, $nocheck, $node) = @_;
2551
2552     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2553
2554     die "unable to find configuration file for VM $vmid - no such machine\n"
2555         if !$nocheck && ! -f $filename;
2556
2557     my $pidfile = pidfile_name($vmid);
2558
2559     if (my $fd = IO::File->new("<$pidfile")) {
2560         my $st = stat($fd);
2561         my $line = <$fd>;
2562         close($fd);
2563
2564         my $mtime = $st->mtime;
2565         if ($mtime > time()) {
2566             warn "file '$filename' modified in future\n";
2567         }
2568
2569         if ($line =~ m/^(\d+)$/) {
2570             my $pid = $1;
2571             if (check_cmdline($pidfile, $pid)) {
2572                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2573                     return $pid;
2574                 }
2575             }
2576         }
2577     }
2578
2579     return undef;
2580 }
2581
2582 sub vzlist {
2583
2584     my $vzlist = config_list();
2585
2586     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2587
2588     while (defined(my $de = $fd->read)) {
2589         next if $de !~ m/^(\d+)\.pid$/;
2590         my $vmid = $1;
2591         next if !defined($vzlist->{$vmid});
2592         if (my $pid = check_running($vmid)) {
2593             $vzlist->{$vmid}->{pid} = $pid;
2594         }
2595     }
2596
2597     return $vzlist;
2598 }
2599
2600 sub disksize {
2601     my ($storecfg, $conf) = @_;
2602
2603     my $bootdisk = $conf->{bootdisk};
2604     return undef if !$bootdisk;
2605     return undef if !is_valid_drivename($bootdisk);
2606
2607     return undef if !$conf->{$bootdisk};
2608
2609     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2610     return undef if !defined($drive);
2611
2612     return undef if drive_is_cdrom($drive);
2613
2614     my $volid = $drive->{file};
2615     return undef if !$volid;
2616
2617     return $drive->{size};
2618 }
2619
2620 my $last_proc_pid_stat;
2621
2622 # get VM status information
2623 # This must be fast and should not block ($full == false)
2624 # We only query KVM using QMP if $full == true (this can be slow)
2625 sub vmstatus {
2626     my ($opt_vmid, $full) = @_;
2627
2628     my $res = {};
2629
2630     my $storecfg = PVE::Storage::config();
2631
2632     my $list = vzlist();
2633     my $defaults = load_defaults();
2634
2635     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2636
2637     my $cpucount = $cpuinfo->{cpus} || 1;
2638
2639     foreach my $vmid (keys %$list) {
2640         next if $opt_vmid && ($vmid ne $opt_vmid);
2641
2642         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2643         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2644
2645         my $d = {};
2646         $d->{pid} = $list->{$vmid}->{pid};
2647
2648         # fixme: better status?
2649         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2650
2651         my $size = disksize($storecfg, $conf);
2652         if (defined($size)) {
2653             $d->{disk} = 0; # no info available
2654             $d->{maxdisk} = $size;
2655         } else {
2656             $d->{disk} = 0;
2657             $d->{maxdisk} = 0;
2658         }
2659
2660         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2661             * ($conf->{cores} || $defaults->{cores});
2662         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2663         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2664
2665         $d->{name} = $conf->{name} || "VM $vmid";
2666         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2667             : $defaults->{memory}*(1024*1024);
2668
2669         if ($conf->{balloon}) {
2670             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2671             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2672                 : $defaults->{shares};
2673         }
2674
2675         $d->{uptime} = 0;
2676         $d->{cpu} = 0;
2677         $d->{mem} = 0;
2678
2679         $d->{netout} = 0;
2680         $d->{netin} = 0;
2681
2682         $d->{diskread} = 0;
2683         $d->{diskwrite} = 0;
2684
2685         $d->{template} = PVE::QemuConfig->is_template($conf);
2686
2687         $d->{serial} = 1 if conf_has_serial($conf);
2688
2689         $res->{$vmid} = $d;
2690     }
2691
2692     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2693     foreach my $dev (keys %$netdev) {
2694         next if $dev !~ m/^tap([1-9]\d*)i/;
2695         my $vmid = $1;
2696         my $d = $res->{$vmid};
2697         next if !$d;
2698
2699         $d->{netout} += $netdev->{$dev}->{receive};
2700         $d->{netin} += $netdev->{$dev}->{transmit};
2701
2702         if ($full) {
2703             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2704             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2705         }
2706
2707     }
2708
2709     my $ctime = gettimeofday;
2710
2711     foreach my $vmid (keys %$list) {
2712
2713         my $d = $res->{$vmid};
2714         my $pid = $d->{pid};
2715         next if !$pid;
2716
2717         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2718         next if !$pstat; # not running
2719
2720         my $used = $pstat->{utime} + $pstat->{stime};
2721
2722         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2723
2724         if ($pstat->{vsize}) {
2725             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2726         }
2727
2728         my $old = $last_proc_pid_stat->{$pid};
2729         if (!$old) {
2730             $last_proc_pid_stat->{$pid} = {
2731                 time => $ctime,
2732                 used => $used,
2733                 cpu => 0,
2734             };
2735             next;
2736         }
2737
2738         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2739
2740         if ($dtime > 1000) {
2741             my $dutime = $used -  $old->{used};
2742
2743             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2744             $last_proc_pid_stat->{$pid} = {
2745                 time => $ctime,
2746                 used => $used,
2747                 cpu => $d->{cpu},
2748             };
2749         } else {
2750             $d->{cpu} = $old->{cpu};
2751         }
2752     }
2753
2754     return $res if !$full;
2755
2756     my $qmpclient = PVE::QMPClient->new();
2757
2758     my $ballooncb = sub {
2759         my ($vmid, $resp) = @_;
2760
2761         my $info = $resp->{'return'};
2762         return if !$info->{max_mem};
2763
2764         my $d = $res->{$vmid};
2765
2766         # use memory assigned to VM
2767         $d->{maxmem} = $info->{max_mem};
2768         $d->{balloon} = $info->{actual};
2769
2770         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2771             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2772             $d->{freemem} = $info->{free_mem};
2773         }
2774
2775         $d->{ballooninfo} = $info;
2776     };
2777
2778     my $blockstatscb = sub {
2779         my ($vmid, $resp) = @_;
2780         my $data = $resp->{'return'} || [];
2781         my $totalrdbytes = 0;
2782         my $totalwrbytes = 0;
2783
2784         for my $blockstat (@$data) {
2785             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2786             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2787
2788             $blockstat->{device} =~ s/drive-//;
2789             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2790         }
2791         $res->{$vmid}->{diskread} = $totalrdbytes;
2792         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2793     };
2794
2795     my $statuscb = sub {
2796         my ($vmid, $resp) = @_;
2797
2798         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2799         # this fails if ballon driver is not loaded, so this must be
2800         # the last commnand (following command are aborted if this fails).
2801         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2802
2803         my $status = 'unknown';
2804         if (!defined($status = $resp->{'return'}->{status})) {
2805             warn "unable to get VM status\n";
2806             return;
2807         }
2808
2809         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2810     };
2811
2812     foreach my $vmid (keys %$list) {
2813         next if $opt_vmid && ($vmid ne $opt_vmid);
2814         next if !$res->{$vmid}->{pid}; # not running
2815         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2816     }
2817
2818     $qmpclient->queue_execute(undef, 2);
2819
2820     foreach my $vmid (keys %$list) {
2821         next if $opt_vmid && ($vmid ne $opt_vmid);
2822         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2823     }
2824
2825     return $res;
2826 }
2827
2828 sub foreach_drive {
2829     my ($conf, $func, @param) = @_;
2830
2831     foreach my $ds (valid_drive_names()) {
2832         next if !defined($conf->{$ds});
2833
2834         my $drive = parse_drive($ds, $conf->{$ds});
2835         next if !$drive;
2836
2837         &$func($ds, $drive, @param);
2838     }
2839 }
2840
2841 sub foreach_volid {
2842     my ($conf, $func, @param) = @_;
2843
2844     my $volhash = {};
2845
2846     my $test_volid = sub {
2847         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
2848
2849         return if !$volid;
2850
2851         $volhash->{$volid}->{cdrom} //= 1;
2852         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2853
2854         $volhash->{$volid}->{replicate} //= 0;
2855         $volhash->{$volid}->{replicate} = 1 if $replicate;
2856
2857         $volhash->{$volid}->{shared} //= 0;
2858         $volhash->{$volid}->{shared} = 1 if $shared;
2859
2860         $volhash->{$volid}->{referenced_in_config} //= 0;
2861         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
2862
2863         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
2864             if defined($snapname);
2865     };
2866
2867     foreach_drive($conf, sub {
2868         my ($ds, $drive) = @_;
2869         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
2870     });
2871
2872     foreach my $snapname (keys %{$conf->{snapshots}}) {
2873         my $snap = $conf->{snapshots}->{$snapname};
2874         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
2875         foreach_drive($snap, sub {
2876             my ($ds, $drive) = @_;
2877             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
2878         });
2879     }
2880
2881     foreach my $volid (keys %$volhash) {
2882         &$func($volid, $volhash->{$volid}, @param);
2883     }
2884 }
2885
2886 sub conf_has_serial {
2887     my ($conf) = @_;
2888
2889     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2890         if ($conf->{"serial$i"}) {
2891             return 1;
2892         }
2893     }
2894
2895     return 0;
2896 }
2897
2898 sub vga_conf_has_spice {
2899     my ($vga) = @_;
2900
2901     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2902
2903     return $1 || 1;
2904 }
2905
2906 sub config_to_command {
2907     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2908
2909     my $cmd = [];
2910     my $globalFlags = [];
2911     my $machineFlags = [];
2912     my $rtcFlags = [];
2913     my $cpuFlags = [];
2914     my $devices = [];
2915     my $pciaddr = '';
2916     my $bridges = {};
2917     my $kvmver = kvm_user_version();
2918     my $vernum = 0; # unknown
2919     my $ostype = $conf->{ostype};
2920     my $winversion = windows_version($ostype);
2921     my $kvm = $conf->{kvm} // 1;
2922
2923     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
2924
2925     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2926         $vernum = $1*1000000+$2*1000;
2927     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2928         $vernum = $1*1000000+$2*1000+$3;
2929     }
2930
2931     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2932
2933     my $have_ovz = -f '/proc/vz/vestat';
2934
2935     my $q35 = machine_type_is_q35($conf);
2936     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2937     my $machine_type = $forcemachine || $conf->{machine};
2938     my $use_old_bios_files = undef;
2939     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2940
2941     my $cpuunits = defined($conf->{cpuunits}) ?
2942             $conf->{cpuunits} : $defaults->{cpuunits};
2943
2944     push @$cmd, '/usr/bin/kvm';
2945
2946     push @$cmd, '-id', $vmid;
2947
2948     my $use_virtio = 0;
2949
2950     my $qmpsocket = qmp_socket($vmid);
2951     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2952     push @$cmd, '-mon', "chardev=qmp,mode=control";
2953
2954
2955     push @$cmd, '-pidfile' , pidfile_name($vmid);
2956
2957     push @$cmd, '-daemonize';
2958
2959     if ($conf->{smbios1}) {
2960         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2961     }
2962
2963     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2964         die "uefi base image not found\n" if ! -f $OVMF_CODE;
2965
2966         my $path;
2967         my $format;
2968         if (my $efidisk = $conf->{efidisk0}) {
2969             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
2970             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2971             $format = $d->{format};
2972             if ($storeid) {
2973                 $path = PVE::Storage::path($storecfg, $d->{file});
2974                 if (!defined($format)) {
2975                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2976                     $format = qemu_img_format($scfg, $volname);
2977                 }
2978             } else {
2979                 $path = $d->{file};
2980                 die "efidisk format must be specified\n"
2981                     if !defined($format);
2982             }
2983         } else {
2984             warn "no efidisk configured! Using temporary efivars disk.\n";
2985             $path = "/tmp/$vmid-ovmf.fd";
2986             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
2987             $format = 'raw';
2988         }
2989
2990         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
2991         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
2992     }
2993
2994
2995     # add usb controllers
2996     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2997     push @$devices, @usbcontrollers if @usbcontrollers;
2998     my $vga = $conf->{vga};
2999
3000     my $qxlnum = vga_conf_has_spice($vga);
3001     $vga = 'qxl' if $qxlnum;
3002
3003     if (!$vga) {
3004         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3005             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3006         } else {
3007             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
3008         }
3009     }
3010
3011     # enable absolute mouse coordinates (needed by vnc)
3012     my $tablet;
3013     if (defined($conf->{tablet})) {
3014         $tablet = $conf->{tablet};
3015     } else {
3016         $tablet = $defaults->{tablet};
3017         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3018         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3019     }
3020
3021     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3022
3023     my $kvm_off = 0;
3024     my $gpu_passthrough;
3025
3026     # host pci devices
3027     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3028         my $d = parse_hostpci($conf->{"hostpci$i"});
3029         next if !$d;
3030
3031         my $pcie = $d->{pcie};
3032         if($pcie){
3033             die "q35 machine model is not enabled" if !$q35;
3034             $pciaddr = print_pcie_addr("hostpci$i");
3035         }else{
3036             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3037         }
3038
3039         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3040         my $romfile = $d->{romfile};
3041
3042         my $xvga = '';
3043         if ($d->{'x-vga'}) {
3044             $xvga = ',x-vga=on';
3045             $kvm_off = 1;
3046             $vga = 'none';
3047             $gpu_passthrough = 1;
3048
3049             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3050                 $xvga = "";
3051             }
3052         }
3053         my $pcidevices = $d->{pciid};
3054         my $multifunction = 1 if @$pcidevices > 1;
3055
3056         my $j=0;
3057         foreach my $pcidevice (@$pcidevices) {
3058
3059             my $id = "hostpci$i";
3060             $id .= ".$j" if $multifunction;
3061             my $addr = $pciaddr;
3062             $addr .= ".$j" if $multifunction;
3063             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3064
3065             if($j == 0){
3066                 $devicestr .= "$rombar$xvga";
3067                 $devicestr .= ",multifunction=on" if $multifunction;
3068                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3069             }
3070
3071             push @$devices, '-device', $devicestr;
3072             $j++;
3073         }
3074     }
3075
3076     # usb devices
3077     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3078     push @$devices, @usbdevices if @usbdevices;
3079     # serial devices
3080     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3081         if (my $path = $conf->{"serial$i"}) {
3082             if ($path eq 'socket') {
3083                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3084                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3085                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3086             } else {
3087                 die "no such serial device\n" if ! -c $path;
3088                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3089                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3090             }
3091         }
3092     }
3093
3094     # parallel devices
3095     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3096         if (my $path = $conf->{"parallel$i"}) {
3097             die "no such parallel device\n" if ! -c $path;
3098             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3099             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3100             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3101         }
3102     }
3103
3104     my $vmname = $conf->{name} || "vm$vmid";
3105
3106     push @$cmd, '-name', $vmname;
3107
3108     my $sockets = 1;
3109     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3110     $sockets = $conf->{sockets} if  $conf->{sockets};
3111
3112     my $cores = $conf->{cores} || 1;
3113
3114     my $maxcpus = $sockets * $cores;
3115
3116     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3117
3118     my $allowed_vcpus = $cpuinfo->{cpus};
3119
3120     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3121         if ($allowed_vcpus < $maxcpus);
3122
3123     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3124
3125         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3126         for (my $i = 2; $i <= $vcpus; $i++)  {
3127             my $cpustr = print_cpu_device($conf,$i);
3128             push @$cmd, '-device', $cpustr;
3129         }
3130
3131     } else {
3132
3133         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3134     }
3135     push @$cmd, '-nodefaults';
3136
3137     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3138
3139     my $bootindex_hash = {};
3140     my $i = 1;
3141     foreach my $o (split(//, $bootorder)) {
3142         $bootindex_hash->{$o} = $i*100;
3143         $i++;
3144     }
3145
3146     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3147
3148     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3149
3150     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3151
3152     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3153
3154     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3155         my $socket = vnc_socket($vmid);
3156         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3157     } else {
3158         push @$cmd, '-nographic';
3159     }
3160
3161     # time drift fix
3162     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3163
3164     my $useLocaltime = $conf->{localtime};
3165
3166     if ($winversion >= 5) { # windows
3167         $useLocaltime = 1 if !defined($conf->{localtime});
3168
3169         # use time drift fix when acpi is enabled
3170         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3171             $tdf = 1 if !defined($conf->{tdf});
3172         }
3173     }
3174
3175     if ($winversion >= 6) {
3176         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3177         push @$cmd, '-no-hpet';
3178     }
3179
3180     push @$rtcFlags, 'driftfix=slew' if $tdf;
3181
3182     if (!$kvm) {
3183         push @$machineFlags, 'accel=tcg';
3184     }
3185
3186     if ($machine_type) {
3187         push @$machineFlags, "type=${machine_type}";
3188     }
3189
3190     if ($conf->{startdate}) {
3191         push @$rtcFlags, "base=$conf->{startdate}";
3192     } elsif ($useLocaltime) {
3193         push @$rtcFlags, 'base=localtime';
3194     }
3195
3196     my $cpu = $kvm ? "kvm64" : "qemu64";
3197     if (my $cputype = $conf->{cpu}) {
3198         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3199             or die "Cannot parse cpu description: $cputype\n";
3200         $cpu = $cpuconf->{cputype};
3201         $kvm_off = 1 if $cpuconf->{hidden};
3202
3203         if (defined(my $flags = $cpuconf->{flags})) {
3204             push @$cpuFlags, split(";", $flags);
3205         }
3206     }
3207
3208     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3209
3210     push @$cpuFlags , '-x2apic'
3211         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3212
3213     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3214
3215     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3216
3217     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3218
3219         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3220         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3221     }
3222
3223     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3224
3225     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3226
3227     push @$cpuFlags, 'kvm=off' if $kvm_off;
3228
3229     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3230         die "internal error"; # should not happen
3231
3232     push @$cpuFlags, "vendor=${cpu_vendor}"
3233         if $cpu_vendor ne 'default';
3234
3235     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3236
3237     push @$cmd, '-cpu', $cpu;
3238
3239     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3240
3241     push @$cmd, '-S' if $conf->{freeze};
3242
3243     # set keyboard layout
3244     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3245     push @$cmd, '-k', $kb if $kb;
3246
3247     # enable sound
3248     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3249     #push @$cmd, '-soundhw', 'es1370';
3250     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3251
3252     if($conf->{agent}) {
3253         my $qgasocket = qmp_socket($vmid, 1);
3254         my $pciaddr = print_pci_addr("qga0", $bridges);
3255         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3256         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3257         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3258     }
3259
3260     my $spice_port;
3261
3262     if ($qxlnum) {
3263         if ($qxlnum > 1) {
3264             if ($winversion){
3265                 for(my $i = 1; $i < $qxlnum; $i++){
3266                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3267                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3268                 }
3269             } else {
3270                 # assume other OS works like Linux
3271                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3272                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3273             }
3274         }
3275
3276         my $pciaddr = print_pci_addr("spice", $bridges);
3277
3278         my $nodename = PVE::INotify::nodename();
3279         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3280         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3281         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3282         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3283         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3284
3285         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3286
3287         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3288         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3289         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3290     }
3291
3292     # enable balloon by default, unless explicitly disabled
3293     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3294         $pciaddr = print_pci_addr("balloon0", $bridges);
3295         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3296     }
3297
3298     if ($conf->{watchdog}) {
3299         my $wdopts = parse_watchdog($conf->{watchdog});
3300         $pciaddr = print_pci_addr("watchdog", $bridges);
3301         my $watchdog = $wdopts->{model} || 'i6300esb';
3302         push @$devices, '-device', "$watchdog$pciaddr";
3303         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3304     }
3305
3306     my $vollist = [];
3307     my $scsicontroller = {};
3308     my $ahcicontroller = {};
3309     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3310
3311     # Add iscsi initiator name if available
3312     if (my $initiator = get_initiator_name()) {
3313         push @$devices, '-iscsi', "initiator-name=$initiator";
3314     }
3315
3316     foreach_drive($conf, sub {
3317         my ($ds, $drive) = @_;
3318
3319         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3320             push @$vollist, $drive->{file};
3321         }
3322
3323         # ignore efidisk here, already added in bios/fw handling code above
3324         return if $drive->{interface} eq 'efidisk';
3325
3326         $use_virtio = 1 if $ds =~ m/^virtio/;
3327
3328         if (drive_is_cdrom ($drive)) {
3329             if ($bootindex_hash->{d}) {
3330                 $drive->{bootindex} = $bootindex_hash->{d};
3331                 $bootindex_hash->{d} += 1;
3332             }
3333         } else {
3334             if ($bootindex_hash->{c}) {
3335                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3336                 $bootindex_hash->{c} += 1;
3337             }
3338         }
3339
3340         if($drive->{interface} eq 'virtio'){
3341            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3342         }
3343
3344         if ($drive->{interface} eq 'scsi') {
3345
3346             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3347
3348             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3349             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3350
3351             my $iothread = '';
3352             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3353                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3354                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3355             } elsif ($drive->{iothread}) {
3356                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3357             }
3358
3359             my $queues = '';
3360             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3361                 $queues = ",num_queues=$drive->{queues}";
3362             }
3363
3364             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3365             $scsicontroller->{$controller}=1;
3366         }
3367
3368         if ($drive->{interface} eq 'sata') {
3369            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3370            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3371            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3372            $ahcicontroller->{$controller}=1;
3373         }
3374
3375         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3376         push @$devices, '-drive',$drive_cmd;
3377         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3378     });
3379
3380     for (my $i = 0; $i < $MAX_NETS; $i++) {
3381          next if !$conf->{"net$i"};
3382          my $d = parse_net($conf->{"net$i"});
3383          next if !$d;
3384
3385          $use_virtio = 1 if $d->{model} eq 'virtio';
3386
3387          if ($bootindex_hash->{n}) {
3388             $d->{bootindex} = $bootindex_hash->{n};
3389             $bootindex_hash->{n} += 1;
3390          }
3391
3392          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3393          push @$devices, '-netdev', $netdevfull;
3394
3395          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3396          push @$devices, '-device', $netdevicefull;
3397     }
3398
3399     if (!$q35) {
3400         # add pci bridges
3401         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3402            $bridges->{1} = 1;
3403            $bridges->{2} = 1;
3404         }
3405
3406         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3407
3408         while (my ($k, $v) = each %$bridges) {
3409             $pciaddr = print_pci_addr("pci.$k");
3410             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3411         }
3412     }
3413
3414     # add custom args
3415     if ($conf->{args}) {
3416         my $aa = PVE::Tools::split_args($conf->{args});
3417         push @$cmd, @$aa;
3418     }
3419
3420     push @$cmd, @$devices;
3421     push @$cmd, '-rtc', join(',', @$rtcFlags)
3422         if scalar(@$rtcFlags);
3423     push @$cmd, '-machine', join(',', @$machineFlags)
3424         if scalar(@$machineFlags);
3425     push @$cmd, '-global', join(',', @$globalFlags)
3426         if scalar(@$globalFlags);
3427
3428     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3429 }
3430
3431 sub vnc_socket {
3432     my ($vmid) = @_;
3433     return "${var_run_tmpdir}/$vmid.vnc";
3434 }
3435
3436 sub spice_port {
3437     my ($vmid) = @_;
3438
3439     my $res = vm_mon_cmd($vmid, 'query-spice');
3440
3441     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3442 }
3443
3444 sub qmp_socket {
3445     my ($vmid, $qga) = @_;
3446     my $sockettype = $qga ? 'qga' : 'qmp';
3447     return "${var_run_tmpdir}/$vmid.$sockettype";
3448 }
3449
3450 sub pidfile_name {
3451     my ($vmid) = @_;
3452     return "${var_run_tmpdir}/$vmid.pid";
3453 }
3454
3455 sub vm_devices_list {
3456     my ($vmid) = @_;
3457
3458     my $res = vm_mon_cmd($vmid, 'query-pci');
3459     my $devices = {};
3460     foreach my $pcibus (@$res) {
3461         foreach my $device (@{$pcibus->{devices}}) {
3462             next if !$device->{'qdev_id'};
3463             if ($device->{'pci_bridge'}) {
3464                 $devices->{$device->{'qdev_id'}} = 1;
3465                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3466                     next if !$bridge_device->{'qdev_id'};
3467                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3468                     $devices->{$device->{'qdev_id'}}++;
3469                 }
3470             } else {
3471                 $devices->{$device->{'qdev_id'}} = 1;
3472             }
3473         }
3474     }
3475
3476     my $resblock = vm_mon_cmd($vmid, 'query-block');
3477     foreach my $block (@$resblock) {
3478         if($block->{device} =~ m/^drive-(\S+)/){
3479                 $devices->{$1} = 1;
3480         }
3481     }
3482
3483     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3484     foreach my $mice (@$resmice) {
3485         if ($mice->{name} eq 'QEMU HID Tablet') {
3486             $devices->{tablet} = 1;
3487             last;
3488         }
3489     }
3490
3491     # for usb devices there is no query-usb
3492     # but we can iterate over the entries in
3493     # qom-list path=/machine/peripheral
3494     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3495     foreach my $per (@$resperipheral) {
3496         if ($per->{name} =~ m/^usb\d+$/) {
3497             $devices->{$per->{name}} = 1;
3498         }
3499     }
3500
3501     return $devices;
3502 }
3503
3504 sub vm_deviceplug {
3505     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3506
3507     my $q35 = machine_type_is_q35($conf);
3508
3509     my $devices_list = vm_devices_list($vmid);
3510     return 1 if defined($devices_list->{$deviceid});
3511
3512     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3513
3514     if ($deviceid eq 'tablet') {
3515
3516         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3517
3518     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3519
3520         die "usb hotplug currently not reliable\n";
3521         # since we can't reliably hot unplug all added usb devices
3522         # and usb passthrough disables live migration
3523         # we disable usb hotplugging for now
3524         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3525
3526     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3527
3528         qemu_iothread_add($vmid, $deviceid, $device);
3529
3530         qemu_driveadd($storecfg, $vmid, $device);
3531         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3532
3533         qemu_deviceadd($vmid, $devicefull);
3534         eval { qemu_deviceaddverify($vmid, $deviceid); };
3535         if (my $err = $@) {
3536             eval { qemu_drivedel($vmid, $deviceid); };
3537             warn $@ if $@;
3538             die $err;
3539         }
3540
3541     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3542
3543
3544         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3545         my $pciaddr = print_pci_addr($deviceid);
3546         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3547
3548         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3549
3550         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3551             qemu_iothread_add($vmid, $deviceid, $device);
3552             $devicefull .= ",iothread=iothread-$deviceid";
3553         }
3554
3555         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3556             $devicefull .= ",num_queues=$device->{queues}";
3557         }
3558
3559         qemu_deviceadd($vmid, $devicefull);
3560         qemu_deviceaddverify($vmid, $deviceid);
3561
3562     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3563
3564         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3565         qemu_driveadd($storecfg, $vmid, $device);
3566
3567         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3568         eval { qemu_deviceadd($vmid, $devicefull); };
3569         if (my $err = $@) {
3570             eval { qemu_drivedel($vmid, $deviceid); };
3571             warn $@ if $@;
3572             die $err;
3573         }
3574
3575     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3576
3577         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3578
3579         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3580         my $use_old_bios_files = undef;
3581         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3582
3583         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3584         qemu_deviceadd($vmid, $netdevicefull);
3585         eval { qemu_deviceaddverify($vmid, $deviceid); };
3586         if (my $err = $@) {
3587             eval { qemu_netdevdel($vmid, $deviceid); };
3588             warn $@ if $@;
3589             die $err;
3590         }
3591
3592     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3593
3594         my $bridgeid = $2;
3595         my $pciaddr = print_pci_addr($deviceid);
3596         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3597
3598         qemu_deviceadd($vmid, $devicefull);
3599         qemu_deviceaddverify($vmid, $deviceid);
3600
3601     } else {
3602         die "can't hotplug device '$deviceid'\n";
3603     }
3604
3605     return 1;
3606 }
3607
3608 # fixme: this should raise exceptions on error!
3609 sub vm_deviceunplug {
3610     my ($vmid, $conf, $deviceid) = @_;
3611
3612     my $devices_list = vm_devices_list($vmid);
3613     return 1 if !defined($devices_list->{$deviceid});
3614
3615     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3616
3617     if ($deviceid eq 'tablet') {
3618
3619         qemu_devicedel($vmid, $deviceid);
3620
3621     } elsif ($deviceid =~ m/^usb\d+$/) {
3622
3623         die "usb hotplug currently not reliable\n";
3624         # when unplugging usb devices this way,
3625         # there may be remaining usb controllers/hubs
3626         # so we disable it for now
3627         qemu_devicedel($vmid, $deviceid);
3628         qemu_devicedelverify($vmid, $deviceid);
3629
3630     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3631
3632         qemu_devicedel($vmid, $deviceid);
3633         qemu_devicedelverify($vmid, $deviceid);
3634         qemu_drivedel($vmid, $deviceid);
3635         qemu_iothread_del($conf, $vmid, $deviceid);
3636
3637     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3638
3639         qemu_devicedel($vmid, $deviceid);
3640         qemu_devicedelverify($vmid, $deviceid);
3641         qemu_iothread_del($conf, $vmid, $deviceid);
3642
3643     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3644
3645         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3646         my $device = parse_drive($deviceid, $conf->{$deviceid});
3647         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3648
3649         qemu_devicedel($vmid, $deviceid);
3650         qemu_drivedel($vmid, $deviceid);
3651         qemu_deletescsihw($conf, $vmid, $deviceid);
3652
3653     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3654
3655         qemu_devicedel($vmid, $deviceid);
3656         qemu_devicedelverify($vmid, $deviceid);
3657         qemu_netdevdel($vmid, $deviceid);
3658
3659     } else {
3660         die "can't unplug device '$deviceid'\n";
3661     }
3662
3663     return 1;
3664 }
3665
3666 sub qemu_deviceadd {
3667     my ($vmid, $devicefull) = @_;
3668
3669     $devicefull = "driver=".$devicefull;
3670     my %options =  split(/[=,]/, $devicefull);
3671
3672     vm_mon_cmd($vmid, "device_add" , %options);
3673 }
3674
3675 sub qemu_devicedel {
3676     my ($vmid, $deviceid) = @_;
3677
3678     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3679 }
3680
3681 sub qemu_iothread_add {
3682     my($vmid, $deviceid, $device) = @_;
3683
3684     if ($device->{iothread}) {
3685         my $iothreads = vm_iothreads_list($vmid);
3686         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3687     }
3688 }
3689
3690 sub qemu_iothread_del {
3691     my($conf, $vmid, $deviceid) = @_;
3692
3693     my $device = parse_drive($deviceid, $conf->{$deviceid});
3694     if ($device->{iothread}) {
3695         my $iothreads = vm_iothreads_list($vmid);
3696         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3697     }
3698 }
3699
3700 sub qemu_objectadd {
3701     my($vmid, $objectid, $qomtype) = @_;
3702
3703     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3704
3705     return 1;
3706 }
3707
3708 sub qemu_objectdel {
3709     my($vmid, $objectid) = @_;
3710
3711     vm_mon_cmd($vmid, "object-del", id => $objectid);
3712
3713     return 1;
3714 }
3715
3716 sub qemu_driveadd {
3717     my ($storecfg, $vmid, $device) = @_;
3718
3719     my $drive = print_drive_full($storecfg, $vmid, $device);
3720     $drive =~ s/\\/\\\\/g;
3721     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3722
3723     # If the command succeeds qemu prints: "OK"
3724     return 1 if $ret =~ m/OK/s;
3725
3726     die "adding drive failed: $ret\n";
3727 }
3728
3729 sub qemu_drivedel {
3730     my($vmid, $deviceid) = @_;
3731
3732     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3733     $ret =~ s/^\s+//;
3734
3735     return 1 if $ret eq "";
3736
3737     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3738     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3739
3740     die "deleting drive $deviceid failed : $ret\n";
3741 }
3742
3743 sub qemu_deviceaddverify {
3744     my ($vmid, $deviceid) = @_;
3745
3746     for (my $i = 0; $i <= 5; $i++) {
3747          my $devices_list = vm_devices_list($vmid);
3748          return 1 if defined($devices_list->{$deviceid});
3749          sleep 1;
3750     }
3751
3752     die "error on hotplug device '$deviceid'\n";
3753 }
3754
3755
3756 sub qemu_devicedelverify {
3757     my ($vmid, $deviceid) = @_;
3758
3759     # need to verify that the device is correctly removed as device_del
3760     # is async and empty return is not reliable
3761
3762     for (my $i = 0; $i <= 5; $i++) {
3763          my $devices_list = vm_devices_list($vmid);
3764          return 1 if !defined($devices_list->{$deviceid});
3765          sleep 1;
3766     }
3767
3768     die "error on hot-unplugging device '$deviceid'\n";
3769 }
3770
3771 sub qemu_findorcreatescsihw {
3772     my ($storecfg, $conf, $vmid, $device) = @_;
3773
3774     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3775
3776     my $scsihwid="$controller_prefix$controller";
3777     my $devices_list = vm_devices_list($vmid);
3778
3779     if(!defined($devices_list->{$scsihwid})) {
3780         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3781     }
3782
3783     return 1;
3784 }
3785
3786 sub qemu_deletescsihw {
3787     my ($conf, $vmid, $opt) = @_;
3788
3789     my $device = parse_drive($opt, $conf->{$opt});
3790
3791     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3792         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3793         return 1;
3794     }
3795
3796     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3797
3798     my $devices_list = vm_devices_list($vmid);
3799     foreach my $opt (keys %{$devices_list}) {
3800         if (PVE::QemuServer::is_valid_drivename($opt)) {
3801             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3802             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3803                 return 1;
3804             }
3805         }
3806     }
3807
3808     my $scsihwid="scsihw$controller";
3809
3810     vm_deviceunplug($vmid, $conf, $scsihwid);
3811
3812     return 1;
3813 }
3814
3815 sub qemu_add_pci_bridge {
3816     my ($storecfg, $conf, $vmid, $device) = @_;
3817
3818     my $bridges = {};
3819
3820     my $bridgeid;
3821
3822     print_pci_addr($device, $bridges);
3823
3824     while (my ($k, $v) = each %$bridges) {
3825         $bridgeid = $k;
3826     }
3827     return 1 if !defined($bridgeid) || $bridgeid < 1;
3828
3829     my $bridge = "pci.$bridgeid";
3830     my $devices_list = vm_devices_list($vmid);
3831
3832     if (!defined($devices_list->{$bridge})) {
3833         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3834     }
3835
3836     return 1;
3837 }
3838
3839 sub qemu_set_link_status {
3840     my ($vmid, $device, $up) = @_;
3841
3842     vm_mon_cmd($vmid, "set_link", name => $device,
3843                up => $up ? JSON::true : JSON::false);
3844 }
3845
3846 sub qemu_netdevadd {
3847     my ($vmid, $conf, $device, $deviceid) = @_;
3848
3849     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3850     my %options =  split(/[=,]/, $netdev);
3851
3852     vm_mon_cmd($vmid, "netdev_add",  %options);
3853     return 1;
3854 }
3855
3856 sub qemu_netdevdel {
3857     my ($vmid, $deviceid) = @_;
3858
3859     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3860 }
3861
3862 sub qemu_usb_hotplug {
3863     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3864
3865     return if !$device;
3866
3867     # remove the old one first
3868     vm_deviceunplug($vmid, $conf, $deviceid);
3869
3870     # check if xhci controller is necessary and available
3871     if ($device->{usb3}) {
3872
3873         my $devicelist = vm_devices_list($vmid);
3874
3875         if (!$devicelist->{xhci}) {
3876             my $pciaddr = print_pci_addr("xhci");
3877             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3878         }
3879     }
3880     my $d = parse_usb_device($device->{host});
3881     $d->{usb3} = $device->{usb3};
3882
3883     # add the new one
3884     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3885 }
3886
3887 sub qemu_cpu_hotplug {
3888     my ($vmid, $conf, $vcpus) = @_;
3889
3890     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3891
3892     my $sockets = 1;
3893     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3894     $sockets = $conf->{sockets} if  $conf->{sockets};
3895     my $cores = $conf->{cores} || 1;
3896     my $maxcpus = $sockets * $cores;
3897
3898     $vcpus = $maxcpus if !$vcpus;
3899
3900     die "you can't add more vcpus than maxcpus\n"
3901         if $vcpus > $maxcpus;
3902
3903     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3904
3905     if ($vcpus < $currentvcpus) {
3906
3907         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3908
3909             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3910                 qemu_devicedel($vmid, "cpu$i");
3911                 my $retry = 0;
3912                 my $currentrunningvcpus = undef;
3913                 while (1) {
3914                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3915                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3916                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3917                     $retry++;
3918                     sleep 1;
3919                 }
3920                 #update conf after each succesfull cpu unplug
3921                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3922                 PVE::QemuConfig->write_config($vmid, $conf);
3923             }
3924         } else {
3925             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3926         }
3927
3928         return;
3929     }
3930
3931     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3932     die "vcpus in running vm does not match its configuration\n"
3933         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3934
3935     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3936
3937         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3938             my $cpustr = print_cpu_device($conf, $i);
3939             qemu_deviceadd($vmid, $cpustr);
3940
3941             my $retry = 0;
3942             my $currentrunningvcpus = undef;
3943             while (1) {
3944                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3945                 last if scalar(@{$currentrunningvcpus}) == $i;
3946                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3947                 sleep 1;
3948                 $retry++;
3949             }
3950             #update conf after each succesfull cpu hotplug
3951             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3952             PVE::QemuConfig->write_config($vmid, $conf);
3953         }
3954     } else {
3955
3956         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3957             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3958         }
3959     }
3960 }
3961
3962 sub qemu_block_set_io_throttle {
3963     my ($vmid, $deviceid,
3964         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3965         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3966         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3967         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3968
3969     return if !check_running($vmid) ;
3970
3971     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3972         bps => int($bps),
3973         bps_rd => int($bps_rd),
3974         bps_wr => int($bps_wr),
3975         iops => int($iops),
3976         iops_rd => int($iops_rd),
3977         iops_wr => int($iops_wr),
3978         bps_max => int($bps_max),
3979         bps_rd_max => int($bps_rd_max),
3980         bps_wr_max => int($bps_wr_max),
3981         iops_max => int($iops_max),
3982         iops_rd_max => int($iops_rd_max),
3983         iops_wr_max => int($iops_wr_max),
3984         bps_max_length => int($bps_max_length),
3985         bps_rd_max_length => int($bps_rd_max_length),
3986         bps_wr_max_length => int($bps_wr_max_length),
3987         iops_max_length => int($iops_max_length),
3988         iops_rd_max_length => int($iops_rd_max_length),
3989         iops_wr_max_length => int($iops_wr_max_length),
3990     );
3991
3992 }
3993
3994 # old code, only used to shutdown old VM after update
3995 sub __read_avail {
3996     my ($fh, $timeout) = @_;
3997
3998     my $sel = new IO::Select;
3999     $sel->add($fh);
4000
4001     my $res = '';
4002     my $buf;
4003
4004     my @ready;
4005     while (scalar (@ready = $sel->can_read($timeout))) {
4006         my $count;
4007         if ($count = $fh->sysread($buf, 8192)) {
4008             if ($buf =~ /^(.*)\(qemu\) $/s) {
4009                 $res .= $1;
4010                 last;
4011             } else {
4012                 $res .= $buf;
4013             }
4014         } else {
4015             if (!defined($count)) {
4016                 die "$!\n";
4017             }
4018             last;
4019         }
4020     }
4021
4022     die "monitor read timeout\n" if !scalar(@ready);
4023
4024     return $res;
4025 }
4026
4027 # old code, only used to shutdown old VM after update
4028 sub vm_monitor_command {
4029     my ($vmid, $cmdstr, $nocheck) = @_;
4030
4031     my $res;
4032
4033     eval {
4034         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4035
4036         my $sname = "${var_run_tmpdir}/$vmid.mon";
4037
4038         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
4039             die "unable to connect to VM $vmid socket - $!\n";
4040
4041         my $timeout = 3;
4042
4043         # hack: migrate sometime blocks the monitor (when migrate_downtime
4044         # is set)
4045         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4046             $timeout = 60*60; # 1 hour
4047         }
4048
4049         # read banner;
4050         my $data = __read_avail($sock, $timeout);
4051
4052         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
4053             die "got unexpected qemu monitor banner\n";
4054         }
4055
4056         my $sel = new IO::Select;
4057         $sel->add($sock);
4058
4059         if (!scalar(my @ready = $sel->can_write($timeout))) {
4060             die "monitor write error - timeout";
4061         }
4062
4063         my $fullcmd = "$cmdstr\r";
4064
4065         # syslog('info', "VM $vmid monitor command: $cmdstr");
4066
4067         my $b;
4068         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
4069             die "monitor write error - $!";
4070         }
4071
4072         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4073
4074         $timeout = 20;
4075
4076         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4077             $timeout = 60*60; # 1 hour
4078         } elsif ($cmdstr =~ m/^(eject|change)/) {
4079             $timeout = 60; # note: cdrom mount command is slow
4080         }
4081         if ($res = __read_avail($sock, $timeout)) {
4082
4083             my @lines = split("\r?\n", $res);
4084
4085             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4086
4087             $res = join("\n", @lines);
4088             $res .= "\n";
4089         }
4090     };
4091
4092     my $err = $@;
4093
4094     if ($err) {
4095         syslog("err", "VM $vmid monitor command failed - $err");
4096         die $err;
4097     }
4098
4099     return $res;
4100 }
4101
4102 sub qemu_block_resize {
4103     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4104
4105     my $running = check_running($vmid);
4106
4107     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4108
4109     return if !$running;
4110
4111     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4112
4113 }
4114
4115 sub qemu_volume_snapshot {
4116     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4117
4118     my $running = check_running($vmid);
4119
4120     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4121         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4122     } else {
4123         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4124     }
4125 }
4126
4127 sub qemu_volume_snapshot_delete {
4128     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4129
4130     my $running = check_running($vmid);
4131
4132     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4133         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4134     } else {
4135         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4136     }
4137 }
4138
4139 sub set_migration_caps {
4140     my ($vmid) = @_;
4141
4142     my $cap_ref = [];
4143
4144     my $enabled_cap = {
4145         "auto-converge" => 1,
4146         "xbzrle" => 1,
4147         "x-rdma-pin-all" => 0,
4148         "zero-blocks" => 0,
4149         "compress" => 0
4150     };
4151
4152     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4153
4154     for my $supported_capability (@$supported_capabilities) {
4155         push @$cap_ref, {
4156             capability => $supported_capability->{capability},
4157             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4158         };
4159     }
4160
4161     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4162 }
4163
4164 my $fast_plug_option = {
4165     'lock' => 1,
4166     'name' => 1,
4167     'onboot' => 1,
4168     'shares' => 1,
4169     'startup' => 1,
4170     'description' => 1,
4171     'protection' => 1,
4172     'vmstatestorage' => 1,
4173 };
4174
4175 # hotplug changes in [PENDING]
4176 # $selection hash can be used to only apply specified options, for
4177 # example: { cores => 1 } (only apply changed 'cores')
4178 # $errors ref is used to return error messages
4179 sub vmconfig_hotplug_pending {
4180     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4181
4182     my $defaults = load_defaults();
4183
4184     # commit values which do not have any impact on running VM first
4185     # Note: those option cannot raise errors, we we do not care about
4186     # $selection and always apply them.
4187
4188     my $add_error = sub {
4189         my ($opt, $msg) = @_;
4190         $errors->{$opt} = "hotplug problem - $msg";
4191     };
4192
4193     my $changes = 0;
4194     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4195         if ($fast_plug_option->{$opt}) {
4196             $conf->{$opt} = $conf->{pending}->{$opt};
4197             delete $conf->{pending}->{$opt};
4198             $changes = 1;
4199         }
4200     }
4201
4202     if ($changes) {
4203         PVE::QemuConfig->write_config($vmid, $conf);
4204         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4205     }
4206
4207     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4208
4209     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4210     while (my ($opt, $force) = each %$pending_delete_hash) {
4211         next if $selection && !$selection->{$opt};
4212         eval {
4213             if ($opt eq 'hotplug') {
4214                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4215             } elsif ($opt eq 'tablet') {
4216                 die "skip\n" if !$hotplug_features->{usb};
4217                 if ($defaults->{tablet}) {
4218                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4219                 } else {
4220                     vm_deviceunplug($vmid, $conf, $opt);
4221                 }
4222             } elsif ($opt =~ m/^usb\d+/) {
4223                 die "skip\n";
4224                 # since we cannot reliably hot unplug usb devices
4225                 # we are disabling it
4226                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4227                 vm_deviceunplug($vmid, $conf, $opt);
4228             } elsif ($opt eq 'vcpus') {
4229                 die "skip\n" if !$hotplug_features->{cpu};
4230                 qemu_cpu_hotplug($vmid, $conf, undef);
4231             } elsif ($opt eq 'balloon') {
4232                 # enable balloon device is not hotpluggable
4233                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4234             } elsif ($fast_plug_option->{$opt}) {
4235                 # do nothing
4236             } elsif ($opt =~ m/^net(\d+)$/) {
4237                 die "skip\n" if !$hotplug_features->{network};
4238                 vm_deviceunplug($vmid, $conf, $opt);
4239             } elsif (is_valid_drivename($opt)) {
4240                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4241                 vm_deviceunplug($vmid, $conf, $opt);
4242                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4243             } elsif ($opt =~ m/^memory$/) {
4244                 die "skip\n" if !$hotplug_features->{memory};
4245                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4246             } elsif ($opt eq 'cpuunits') {
4247                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4248             } elsif ($opt eq 'cpulimit') {
4249                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4250             } else {
4251                 die "skip\n";
4252             }
4253         };
4254         if (my $err = $@) {
4255             &$add_error($opt, $err) if $err ne "skip\n";
4256         } else {
4257             # save new config if hotplug was successful
4258             delete $conf->{$opt};
4259             vmconfig_undelete_pending_option($conf, $opt);
4260             PVE::QemuConfig->write_config($vmid, $conf);
4261             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4262         }
4263     }
4264
4265     foreach my $opt (keys %{$conf->{pending}}) {
4266         next if $selection && !$selection->{$opt};
4267         my $value = $conf->{pending}->{$opt};
4268         eval {
4269             if ($opt eq 'hotplug') {
4270                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4271             } elsif ($opt eq 'tablet') {
4272                 die "skip\n" if !$hotplug_features->{usb};
4273                 if ($value == 1) {
4274                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4275                 } elsif ($value == 0) {
4276                     vm_deviceunplug($vmid, $conf, $opt);
4277                 }
4278             } elsif ($opt =~ m/^usb\d+$/) {
4279                 die "skip\n";
4280                 # since we cannot reliably hot unplug usb devices
4281                 # we are disabling it
4282                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4283                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4284                 die "skip\n" if !$d;
4285                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4286             } elsif ($opt eq 'vcpus') {
4287                 die "skip\n" if !$hotplug_features->{cpu};
4288                 qemu_cpu_hotplug($vmid, $conf, $value);
4289             } elsif ($opt eq 'balloon') {
4290                 # enable/disable balloning device is not hotpluggable
4291                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4292                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4293                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4294
4295                 # allow manual ballooning if shares is set to zero
4296                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4297                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4298                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4299                 }
4300             } elsif ($opt =~ m/^net(\d+)$/) {
4301                 # some changes can be done without hotplug
4302                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4303                                     $vmid, $opt, $value);
4304             } elsif (is_valid_drivename($opt)) {
4305                 # some changes can be done without hotplug
4306                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4307                                      $vmid, $opt, $value, 1);
4308             } elsif ($opt =~ m/^memory$/) { #dimms
4309                 die "skip\n" if !$hotplug_features->{memory};
4310                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4311             } elsif ($opt eq 'cpuunits') {
4312                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4313             } elsif ($opt eq 'cpulimit') {
4314                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4315                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4316             } else {
4317                 die "skip\n";  # skip non-hot-pluggable options
4318             }
4319         };
4320         if (my $err = $@) {
4321             &$add_error($opt, $err) if $err ne "skip\n";
4322         } else {
4323             # save new config if hotplug was successful
4324             $conf->{$opt} = $value;
4325             delete $conf->{pending}->{$opt};
4326             PVE::QemuConfig->write_config($vmid, $conf);
4327             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4328         }
4329     }
4330 }
4331
4332 sub try_deallocate_drive {
4333     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4334
4335     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4336         my $volid = $drive->{file};
4337         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4338             my $sid = PVE::Storage::parse_volume_id($volid);
4339             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4340
4341             # check if the disk is really unused
4342             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4343                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4344             PVE::Storage::vdisk_free($storecfg, $volid);
4345             return 1;
4346         } else {
4347             # If vm is not owner of this disk remove from config
4348             return 1;
4349         }
4350     }
4351
4352     return undef;
4353 }
4354
4355 sub vmconfig_delete_or_detach_drive {
4356     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4357
4358     my $drive = parse_drive($opt, $conf->{$opt});
4359
4360     my $rpcenv = PVE::RPCEnvironment::get();
4361     my $authuser = $rpcenv->get_user();
4362
4363     if ($force) {
4364         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4365         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4366     } else {
4367         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4368     }
4369 }
4370
4371 sub vmconfig_apply_pending {
4372     my ($vmid, $conf, $storecfg) = @_;
4373
4374     # cold plug
4375
4376     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4377     while (my ($opt, $force) = each %$pending_delete_hash) {
4378         die "internal error" if $opt =~ m/^unused/;
4379         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4380         if (!defined($conf->{$opt})) {
4381             vmconfig_undelete_pending_option($conf, $opt);
4382             PVE::QemuConfig->write_config($vmid, $conf);
4383         } elsif (is_valid_drivename($opt)) {
4384             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4385             vmconfig_undelete_pending_option($conf, $opt);
4386             delete $conf->{$opt};
4387             PVE::QemuConfig->write_config($vmid, $conf);
4388         } else {
4389             vmconfig_undelete_pending_option($conf, $opt);
4390             delete $conf->{$opt};
4391             PVE::QemuConfig->write_config($vmid, $conf);
4392         }
4393     }
4394
4395     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4396
4397     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4398         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4399
4400         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4401             # skip if nothing changed
4402         } elsif (is_valid_drivename($opt)) {
4403             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4404                 if defined($conf->{$opt});
4405             $conf->{$opt} = $conf->{pending}->{$opt};
4406         } else {
4407             $conf->{$opt} = $conf->{pending}->{$opt};
4408         }
4409
4410         delete $conf->{pending}->{$opt};
4411         PVE::QemuConfig->write_config($vmid, $conf);
4412     }
4413 }
4414
4415 my $safe_num_ne = sub {
4416     my ($a, $b) = @_;
4417
4418     return 0 if !defined($a) && !defined($b);
4419     return 1 if !defined($a);
4420     return 1 if !defined($b);
4421
4422     return $a != $b;
4423 };
4424
4425 my $safe_string_ne = sub {
4426     my ($a, $b) = @_;
4427
4428     return 0 if !defined($a) && !defined($b);
4429     return 1 if !defined($a);
4430     return 1 if !defined($b);
4431
4432     return $a ne $b;
4433 };
4434
4435 sub vmconfig_update_net {
4436     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4437
4438     my $newnet = parse_net($value);
4439
4440     if ($conf->{$opt}) {
4441         my $oldnet = parse_net($conf->{$opt});
4442
4443         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4444             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4445             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4446             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4447
4448             # for non online change, we try to hot-unplug
4449             die "skip\n" if !$hotplug;
4450             vm_deviceunplug($vmid, $conf, $opt);
4451         } else {
4452
4453             die "internal error" if $opt !~ m/net(\d+)/;
4454             my $iface = "tap${vmid}i$1";
4455
4456             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4457                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4458                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4459                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4460                 PVE::Network::tap_unplug($iface);
4461                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4462             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4463                 # Rate can be applied on its own but any change above needs to
4464                 # include the rate in tap_plug since OVS resets everything.
4465                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4466             }
4467
4468             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4469                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4470             }
4471
4472             return 1;
4473         }
4474     }
4475
4476     if ($hotplug) {
4477         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4478     } else {
4479         die "skip\n";
4480     }
4481 }
4482
4483 sub vmconfig_update_disk {
4484     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4485
4486     # fixme: do we need force?
4487
4488     my $drive = parse_drive($opt, $value);
4489
4490     if ($conf->{$opt}) {
4491
4492         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4493
4494             my $media = $drive->{media} || 'disk';
4495             my $oldmedia = $old_drive->{media} || 'disk';
4496             die "unable to change media type\n" if $media ne $oldmedia;
4497
4498             if (!drive_is_cdrom($old_drive)) {
4499
4500                 if ($drive->{file} ne $old_drive->{file}) {
4501
4502                     die "skip\n" if !$hotplug;
4503
4504                     # unplug and register as unused
4505                     vm_deviceunplug($vmid, $conf, $opt);
4506                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4507
4508                 } else {
4509                     # update existing disk
4510
4511                     # skip non hotpluggable value
4512                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4513                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4514                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4515                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4516                         die "skip\n";
4517                     }
4518
4519                     # apply throttle
4520                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4521                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4522                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4523                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4524                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4525                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4526                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4527                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4528                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4529                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4530                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4531                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4532                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4533                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4534                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4535                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4536                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4537                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4538
4539                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4540                                                    ($drive->{mbps} || 0)*1024*1024,
4541                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4542                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4543                                                    $drive->{iops} || 0,
4544                                                    $drive->{iops_rd} || 0,
4545                                                    $drive->{iops_wr} || 0,
4546                                                    ($drive->{mbps_max} || 0)*1024*1024,
4547                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4548                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4549                                                    $drive->{iops_max} || 0,
4550                                                    $drive->{iops_rd_max} || 0,
4551                                                    $drive->{iops_wr_max} || 0,
4552                                                    $drive->{bps_max_length} || 1,
4553                                                    $drive->{bps_rd_max_length} || 1,
4554                                                    $drive->{bps_wr_max_length} || 1,
4555                                                    $drive->{iops_max_length} || 1,
4556                                                    $drive->{iops_rd_max_length} || 1,
4557                                                    $drive->{iops_wr_max_length} || 1);
4558
4559                     }
4560
4561                     return 1;
4562                 }
4563
4564             } else { # cdrom
4565
4566                 if ($drive->{file} eq 'none') {
4567                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4568                 } else {
4569                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4570                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4571                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4572                 }
4573
4574                 return 1;
4575             }
4576         }
4577     }
4578
4579     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4580     # hotplug new disks
4581     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4582     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4583 }
4584
4585 sub vm_start {
4586     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4587         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4588
4589     PVE::QemuConfig->lock_config($vmid, sub {
4590         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4591
4592         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4593
4594         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4595
4596         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4597
4598         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4599             vmconfig_apply_pending($vmid, $conf, $storecfg);
4600             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4601         }
4602
4603         my $defaults = load_defaults();
4604
4605         # set environment variable useful inside network script
4606         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4607
4608         my $local_volumes = {};
4609
4610         if ($targetstorage) {
4611             foreach_drive($conf, sub {
4612                 my ($ds, $drive) = @_;
4613
4614                 return if drive_is_cdrom($drive);
4615
4616                 my $volid = $drive->{file};
4617
4618                 return if !$volid;
4619
4620                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4621
4622                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4623                 return if $scfg->{shared};
4624                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4625             });
4626
4627             my $format = undef;
4628
4629             foreach my $opt (sort keys %$local_volumes) {
4630
4631                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4632                 my $drive = parse_drive($opt, $conf->{$opt});
4633
4634                 #if remote storage is specified, use default format
4635                 if ($targetstorage && $targetstorage ne "1") {
4636                     $storeid = $targetstorage;
4637                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4638                     $format = $defFormat;
4639                 } else {
4640                     #else we use same format than original
4641                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4642                     $format = qemu_img_format($scfg, $volid);
4643                 }
4644
4645                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4646                 my $newdrive = $drive;
4647                 $newdrive->{format} = $format;
4648                 $newdrive->{file} = $newvolid;
4649                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4650                 $local_volumes->{$opt} = $drivestr;
4651                 #pass drive to conf for command line
4652                 $conf->{$opt} = $drivestr;
4653             }
4654         }
4655
4656         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4657
4658         my $migrate_port = 0;
4659         my $migrate_uri;
4660         if ($statefile) {
4661             if ($statefile eq 'tcp') {
4662                 my $localip = "localhost";
4663                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4664                 my $nodename = PVE::INotify::nodename();
4665
4666                 if (!defined($migration_type)) {
4667                     if (defined($datacenterconf->{migration}->{type})) {
4668                         $migration_type = $datacenterconf->{migration}->{type};
4669                     } else {
4670                         $migration_type = 'secure';
4671                     }
4672                 }
4673
4674                 if ($migration_type eq 'insecure') {
4675                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4676                     if ($migrate_network_addr) {
4677                         $localip = $migrate_network_addr;
4678                     } else {
4679                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4680                     }
4681
4682                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4683                 }
4684
4685                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4686                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4687                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4688                 push @$cmd, '-incoming', $migrate_uri;
4689                 push @$cmd, '-S';
4690
4691             } elsif ($statefile eq 'unix') {
4692                 # should be default for secure migrations as a ssh TCP forward
4693                 # tunnel is not deterministic reliable ready and fails regurarly
4694                 # to set up in time, so use UNIX socket forwards
4695                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4696                 unlink $socket_addr;
4697
4698                 $migrate_uri = "unix:$socket_addr";
4699
4700                 push @$cmd, '-incoming', $migrate_uri;
4701                 push @$cmd, '-S';
4702
4703             } else {
4704                 push @$cmd, '-loadstate', $statefile;
4705             }
4706         } elsif ($paused) {
4707             push @$cmd, '-S';
4708         }
4709
4710         # host pci devices
4711         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4712           my $d = parse_hostpci($conf->{"hostpci$i"});
4713           next if !$d;
4714           my $pcidevices = $d->{pciid};
4715           foreach my $pcidevice (@$pcidevices) {
4716                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4717
4718                 my $info = pci_device_info("0000:$pciid");
4719                 die "IOMMU not present\n" if !check_iommu_support();
4720                 die "no pci device info for device '$pciid'\n" if !$info;
4721                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4722                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4723           }
4724         }
4725
4726         PVE::Storage::activate_volumes($storecfg, $vollist);
4727
4728         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4729             my $cmd = [];
4730             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4731             eval  { run_command($cmd); };
4732         }
4733
4734         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4735                                                   : $defaults->{cpuunits};
4736
4737         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4738         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4739
4740         my %properties = (
4741             Slice => 'qemu.slice',
4742             KillMode => 'none',
4743             CPUShares => $cpuunits
4744         );
4745
4746         if (my $cpulimit = $conf->{cpulimit}) {
4747             $properties{CPUQuota} = int($cpulimit * 100);
4748         }
4749         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4750
4751         if ($conf->{hugepages}) {
4752
4753             my $code = sub {
4754                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4755                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4756
4757                 PVE::QemuServer::Memory::hugepages_mount();
4758                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4759
4760                 eval  {
4761                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4762                     run_command($cmd, %run_params);
4763                 };
4764
4765                 if (my $err = $@) {
4766                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4767                     die $err;
4768                 }
4769
4770                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4771             };
4772             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4773
4774         } else {
4775             eval  {
4776                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4777                 run_command($cmd, %run_params);
4778             };
4779         }
4780
4781         if (my $err = $@) {
4782             # deactivate volumes if start fails
4783             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4784             die "start failed: $err";
4785         }
4786
4787         print "migration listens on $migrate_uri\n" if $migrate_uri;
4788
4789         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4790             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4791             warn $@ if $@;
4792         }
4793
4794         #start nbd server for storage migration
4795         if ($targetstorage) {
4796             my $nodename = PVE::INotify::nodename();
4797             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4798             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4799             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4800             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4801
4802             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4803
4804             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4805
4806             foreach my $opt (sort keys %$local_volumes) {
4807                 my $volid = $local_volumes->{$opt};
4808                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4809                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4810                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4811             }
4812         }
4813
4814         if ($migratedfrom) {
4815             eval {
4816                 set_migration_caps($vmid);
4817             };
4818             warn $@ if $@;
4819
4820             if ($spice_port) {
4821                 print "spice listens on port $spice_port\n";
4822                 if ($spice_ticket) {
4823                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4824                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4825                 }
4826             }
4827
4828         } else {
4829             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4830                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4831                     if $conf->{balloon};
4832             }
4833
4834             foreach my $opt (keys %$conf) {
4835                 next if $opt !~  m/^net\d+$/;
4836                 my $nicconf = parse_net($conf->{$opt});
4837                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4838             }
4839         }
4840
4841         vm_mon_cmd_nocheck($vmid, 'qom-set',
4842                     path => "machine/peripheral/balloon0",
4843                     property => "guest-stats-polling-interval",
4844                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4845
4846     });
4847 }
4848
4849 sub vm_mon_cmd {
4850     my ($vmid, $execute, %params) = @_;
4851
4852     my $cmd = { execute => $execute, arguments => \%params };
4853     vm_qmp_command($vmid, $cmd);
4854 }
4855
4856 sub vm_mon_cmd_nocheck {
4857     my ($vmid, $execute, %params) = @_;
4858
4859     my $cmd = { execute => $execute, arguments => \%params };
4860     vm_qmp_command($vmid, $cmd, 1);
4861 }
4862
4863 sub vm_qmp_command {
4864     my ($vmid, $cmd, $nocheck) = @_;
4865
4866     my $res;
4867
4868     my $timeout;
4869     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4870         $timeout = $cmd->{arguments}->{timeout};
4871         delete $cmd->{arguments}->{timeout};
4872     }
4873
4874     eval {
4875         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4876         my $sname = qmp_socket($vmid);
4877         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4878             my $qmpclient = PVE::QMPClient->new();
4879
4880             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4881         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4882             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4883                 if scalar(%{$cmd->{arguments}});
4884             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4885         } else {
4886             die "unable to open monitor socket\n";
4887         }
4888     };
4889     if (my $err = $@) {
4890         syslog("err", "VM $vmid qmp command failed - $err");
4891         die $err;
4892     }
4893
4894     return $res;
4895 }
4896
4897 sub vm_human_monitor_command {
4898     my ($vmid, $cmdline) = @_;
4899
4900     my $res;
4901
4902     my $cmd = {
4903         execute => 'human-monitor-command',
4904         arguments => { 'command-line' => $cmdline},
4905     };
4906
4907     return vm_qmp_command($vmid, $cmd);
4908 }
4909
4910 sub vm_commandline {
4911     my ($storecfg, $vmid) = @_;
4912
4913     my $conf = PVE::QemuConfig->load_config($vmid);
4914
4915     my $defaults = load_defaults();
4916
4917     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4918
4919     return PVE::Tools::cmd2string($cmd);
4920 }
4921
4922 sub vm_reset {
4923     my ($vmid, $skiplock) = @_;
4924
4925     PVE::QemuConfig->lock_config($vmid, sub {
4926
4927         my $conf = PVE::QemuConfig->load_config($vmid);
4928
4929         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4930
4931         vm_mon_cmd($vmid, "system_reset");
4932     });
4933 }
4934
4935 sub get_vm_volumes {
4936     my ($conf) = @_;
4937
4938     my $vollist = [];
4939     foreach_volid($conf, sub {
4940         my ($volid, $attr) = @_;
4941
4942         return if $volid =~ m|^/|;
4943
4944         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4945         return if !$sid;
4946
4947         push @$vollist, $volid;
4948     });
4949
4950     return $vollist;
4951 }
4952
4953 sub vm_stop_cleanup {
4954     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4955
4956     eval {
4957
4958         if (!$keepActive) {
4959             my $vollist = get_vm_volumes($conf);
4960             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4961         }
4962
4963         foreach my $ext (qw(mon qmp pid vnc qga)) {
4964             unlink "/var/run/qemu-server/${vmid}.$ext";
4965         }
4966
4967         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4968     };
4969     warn $@ if $@; # avoid errors - just warn
4970 }
4971
4972 # Note: use $nockeck to skip tests if VM configuration file exists.
4973 # We need that when migration VMs to other nodes (files already moved)
4974 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4975 sub vm_stop {
4976     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4977
4978     $force = 1 if !defined($force) && !$shutdown;
4979
4980     if ($migratedfrom){
4981         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4982         kill 15, $pid if $pid;
4983         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4984         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4985         return;
4986     }
4987
4988     PVE::QemuConfig->lock_config($vmid, sub {
4989
4990         my $pid = check_running($vmid, $nocheck);
4991         return if !$pid;
4992
4993         my $conf;
4994         if (!$nocheck) {
4995             $conf = PVE::QemuConfig->load_config($vmid);
4996             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4997             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4998                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4999                 $timeout = $opts->{down} if $opts->{down};
5000             }
5001         }
5002
5003         $timeout = 60 if !defined($timeout);
5004
5005         eval {
5006             if ($shutdown) {
5007                 if (defined($conf) && $conf->{agent}) {
5008                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5009                 } else {
5010                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5011                 }
5012             } else {
5013                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5014             }
5015         };
5016         my $err = $@;
5017
5018         if (!$err) {
5019             my $count = 0;
5020             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5021                 $count++;
5022                 sleep 1;
5023             }
5024
5025             if ($count >= $timeout) {
5026                 if ($force) {
5027                     warn "VM still running - terminating now with SIGTERM\n";
5028                     kill 15, $pid;
5029                 } else {
5030                     die "VM quit/powerdown failed - got timeout\n";
5031                 }
5032             } else {
5033                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5034                 return;
5035             }
5036         } else {
5037             if ($force) {
5038                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5039                 kill 15, $pid;
5040             } else {
5041                 die "VM quit/powerdown failed\n";
5042             }
5043         }
5044
5045         # wait again
5046         $timeout = 10;
5047
5048         my $count = 0;
5049         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5050             $count++;
5051             sleep 1;
5052         }
5053
5054         if ($count >= $timeout) {
5055             warn "VM still running - terminating now with SIGKILL\n";
5056             kill 9, $pid;
5057             sleep 1;
5058         }
5059
5060         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5061    });
5062 }
5063
5064 sub vm_suspend {
5065     my ($vmid, $skiplock) = @_;
5066
5067     PVE::QemuConfig->lock_config($vmid, sub {
5068
5069         my $conf = PVE::QemuConfig->load_config($vmid);
5070
5071         PVE::QemuConfig->check_lock($conf)
5072             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5073
5074         vm_mon_cmd($vmid, "stop");
5075     });
5076 }
5077
5078 sub vm_resume {
5079     my ($vmid, $skiplock, $nocheck) = @_;
5080
5081     PVE::QemuConfig->lock_config($vmid, sub {
5082
5083         if (!$nocheck) {
5084
5085             my $conf = PVE::QemuConfig->load_config($vmid);
5086
5087             PVE::QemuConfig->check_lock($conf)
5088                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5089
5090             vm_mon_cmd($vmid, "cont");
5091
5092         } else {
5093             vm_mon_cmd_nocheck($vmid, "cont");
5094         }
5095     });
5096 }
5097
5098 sub vm_sendkey {
5099     my ($vmid, $skiplock, $key) = @_;
5100
5101     PVE::QemuConfig->lock_config($vmid, sub {
5102
5103         my $conf = PVE::QemuConfig->load_config($vmid);
5104
5105         # there is no qmp command, so we use the human monitor command
5106         vm_human_monitor_command($vmid, "sendkey $key");
5107     });
5108 }
5109
5110 sub vm_destroy {
5111     my ($storecfg, $vmid, $skiplock) = @_;
5112
5113     PVE::QemuConfig->lock_config($vmid, sub {
5114
5115         my $conf = PVE::QemuConfig->load_config($vmid);
5116
5117         if (!check_running($vmid)) {
5118             destroy_vm($storecfg, $vmid, undef, $skiplock);
5119         } else {
5120             die "VM $vmid is running - destroy failed\n";
5121         }
5122     });
5123 }
5124
5125 # pci helpers
5126
5127 sub file_write {
5128     my ($filename, $buf) = @_;
5129
5130     my $fh = IO::File->new($filename, "w");
5131     return undef if !$fh;
5132
5133     my $res = print $fh $buf;
5134
5135     $fh->close();
5136
5137     return $res;
5138 }
5139
5140 sub pci_device_info {
5141     my ($name) = @_;
5142
5143     my $res;
5144
5145     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5146     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5147
5148     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5149     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5150
5151     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5152     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5153
5154     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5155     return undef if !defined($product) || $product !~ s/^0x//;
5156
5157     $res = {
5158         name => $name,
5159         vendor => $vendor,
5160         product => $product,
5161         domain => $domain,
5162         bus => $bus,
5163         slot => $slot,
5164         func => $func,
5165         irq => $irq,
5166         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5167     };
5168
5169     return $res;
5170 }
5171
5172 sub pci_dev_reset {
5173     my ($dev) = @_;
5174
5175     my $name = $dev->{name};
5176
5177     my $fn = "$pcisysfs/devices/$name/reset";
5178
5179     return file_write($fn, "1");
5180 }
5181
5182 sub pci_dev_bind_to_vfio {
5183     my ($dev) = @_;
5184
5185     my $name = $dev->{name};
5186
5187     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5188
5189     if (!-d $vfio_basedir) {
5190         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5191     }
5192     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5193
5194     my $testdir = "$vfio_basedir/$name";
5195     return 1 if -d $testdir;
5196
5197     my $data = "$dev->{vendor} $dev->{product}";
5198     return undef if !file_write("$vfio_basedir/new_id", $data);
5199
5200     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5201     if (!file_write($fn, $name)) {
5202         return undef if -f $fn;
5203     }
5204
5205     $fn = "$vfio_basedir/bind";
5206     if (! -d $testdir) {
5207         return undef if !file_write($fn, $name);
5208     }
5209
5210     return -d $testdir;
5211 }
5212
5213 sub pci_dev_group_bind_to_vfio {
5214     my ($pciid) = @_;
5215
5216     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5217
5218     if (!-d $vfio_basedir) {
5219         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5220     }
5221     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5222
5223     # get IOMMU group devices
5224     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5225       my @devs = grep /^0000:/, readdir($D);
5226     closedir($D);
5227
5228     foreach my $pciid (@devs) {
5229         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5230
5231         # pci bridges, switches or root ports are not supported
5232         # they have a pci_bus subdirectory so skip them
5233         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5234
5235         my $info = pci_device_info($1);
5236         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5237     }
5238
5239     return 1;
5240 }
5241
5242 # vzdump restore implementaion
5243
5244 sub tar_archive_read_firstfile {
5245     my $archive = shift;
5246
5247     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5248
5249     # try to detect archive type first
5250     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5251         die "unable to open file '$archive'\n";
5252     my $firstfile = <$fh>;
5253     kill 15, $pid;
5254     close $fh;
5255
5256     die "ERROR: archive contaions no data\n" if !$firstfile;
5257     chomp $firstfile;
5258
5259     return $firstfile;
5260 }
5261
5262 sub tar_restore_cleanup {
5263     my ($storecfg, $statfile) = @_;
5264
5265     print STDERR "starting cleanup\n";
5266
5267     if (my $fd = IO::File->new($statfile, "r")) {
5268         while (defined(my $line = <$fd>)) {
5269             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5270                 my $volid = $2;
5271                 eval {
5272                     if ($volid =~ m|^/|) {
5273                         unlink $volid || die 'unlink failed\n';
5274                     } else {
5275                         PVE::Storage::vdisk_free($storecfg, $volid);
5276                     }
5277                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5278                 };
5279                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5280             } else {
5281                 print STDERR "unable to parse line in statfile - $line";
5282             }
5283         }
5284         $fd->close();
5285     }
5286 }
5287
5288 sub restore_archive {
5289     my ($archive, $vmid, $user, $opts) = @_;
5290
5291     my $format = $opts->{format};
5292     my $comp;
5293
5294     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5295         $format = 'tar' if !$format;
5296         $comp = 'gzip';
5297     } elsif ($archive =~ m/\.tar$/) {
5298         $format = 'tar' if !$format;
5299     } elsif ($archive =~ m/.tar.lzo$/) {
5300         $format = 'tar' if !$format;
5301         $comp = 'lzop';
5302     } elsif ($archive =~ m/\.vma$/) {
5303         $format = 'vma' if !$format;
5304     } elsif ($archive =~ m/\.vma\.gz$/) {
5305         $format = 'vma' if !$format;
5306         $comp = 'gzip';
5307     } elsif ($archive =~ m/\.vma\.lzo$/) {
5308         $format = 'vma' if !$format;
5309         $comp = 'lzop';
5310     } else {
5311         $format = 'vma' if !$format; # default
5312     }
5313
5314     # try to detect archive format
5315     if ($format eq 'tar') {
5316         return restore_tar_archive($archive, $vmid, $user, $opts);
5317     } else {
5318         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5319     }
5320 }
5321
5322 sub restore_update_config_line {
5323     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5324
5325     return if $line =~ m/^\#qmdump\#/;
5326     return if $line =~ m/^\#vzdump\#/;
5327     return if $line =~ m/^lock:/;
5328     return if $line =~ m/^unused\d+:/;
5329     return if $line =~ m/^parent:/;
5330     return if $line =~ m/^template:/; # restored VM is never a template
5331
5332     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5333     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5334         # try to convert old 1.X settings
5335         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5336         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5337             my ($model, $macaddr) = split(/\=/, $devconfig);
5338             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5339             my $net = {
5340                 model => $model,
5341                 bridge => "vmbr$ind",
5342                 macaddr => $macaddr,
5343             };
5344             my $netstr = print_net($net);
5345
5346             print $outfd "net$cookie->{netcount}: $netstr\n";
5347             $cookie->{netcount}++;
5348         }
5349     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5350         my ($id, $netstr) = ($1, $2);
5351         my $net = parse_net($netstr);
5352         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5353         $netstr = print_net($net);
5354         print $outfd "$id: $netstr\n";
5355     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5356         my $virtdev = $1;
5357         my $value = $3;
5358         my $di = parse_drive($virtdev, $value);
5359         if (defined($di->{backup}) && !$di->{backup}) {
5360             print $outfd "#$line";
5361         } elsif ($map->{$virtdev}) {
5362             delete $di->{format}; # format can change on restore
5363             $di->{file} = $map->{$virtdev};
5364             $value = print_drive($vmid, $di);
5365             print $outfd "$virtdev: $value\n";
5366         } else {
5367             print $outfd $line;
5368         }
5369     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5370         my ($uuid, $uuid_str);
5371         UUID::generate($uuid);
5372         UUID::unparse($uuid, $uuid_str);
5373         my $smbios1 = parse_smbios1($2);
5374         $smbios1->{uuid} = $uuid_str;
5375         print $outfd $1.print_smbios1($smbios1)."\n";
5376     } else {
5377         print $outfd $line;
5378     }
5379 }
5380
5381 sub scan_volids {
5382     my ($cfg, $vmid) = @_;
5383
5384     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5385
5386     my $volid_hash = {};
5387     foreach my $storeid (keys %$info) {
5388         foreach my $item (@{$info->{$storeid}}) {
5389             next if !($item->{volid} && $item->{size});
5390             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5391             $volid_hash->{$item->{volid}} = $item;
5392         }
5393     }
5394
5395     return $volid_hash;
5396 }
5397
5398 sub is_volume_in_use {
5399     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5400
5401     my $path = PVE::Storage::path($storecfg, $volid);
5402
5403     my $scan_config = sub {
5404         my ($cref, $snapname) = @_;
5405
5406         foreach my $key (keys %$cref) {
5407             my $value = $cref->{$key};
5408             if (is_valid_drivename($key)) {
5409                 next if $skip_drive && $key eq $skip_drive;
5410                 my $drive = parse_drive($key, $value);
5411                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5412                 return 1 if $volid eq $drive->{file};
5413                 if ($drive->{file} =~ m!^/!) {
5414                     return 1 if $drive->{file} eq $path;
5415                 } else {
5416                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5417                     next if !$storeid;
5418                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5419                     next if !$scfg;
5420                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5421                 }
5422             }
5423         }
5424
5425         return 0;
5426     };
5427
5428     return 1 if &$scan_config($conf);
5429
5430     undef $skip_drive;
5431
5432     foreach my $snapname (keys %{$conf->{snapshots}}) {
5433         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5434     }
5435
5436     return 0;
5437 }
5438
5439 sub update_disksize {
5440     my ($vmid, $conf, $volid_hash) = @_;
5441
5442     my $changes;
5443
5444     # used and unused disks
5445     my $referenced = {};
5446
5447     # Note: it is allowed to define multiple storages with same path (alias), so
5448     # we need to check both 'volid' and real 'path' (two different volid can point
5449     # to the same path).
5450
5451     my $referencedpath = {};
5452
5453     # update size info
5454     foreach my $opt (keys %$conf) {
5455         if (is_valid_drivename($opt)) {
5456             my $drive = parse_drive($opt, $conf->{$opt});
5457             my $volid = $drive->{file};
5458             next if !$volid;
5459
5460             $referenced->{$volid} = 1;
5461             if ($volid_hash->{$volid} &&
5462                 (my $path = $volid_hash->{$volid}->{path})) {
5463                 $referencedpath->{$path} = 1;
5464             }
5465
5466             next if drive_is_cdrom($drive);
5467             next if !$volid_hash->{$volid};
5468
5469             $drive->{size} = $volid_hash->{$volid}->{size};
5470             my $new = print_drive($vmid, $drive);
5471             if ($new ne $conf->{$opt}) {
5472                 $changes = 1;
5473                 $conf->{$opt} = $new;
5474             }
5475         }
5476     }
5477
5478     # remove 'unusedX' entry if volume is used
5479     foreach my $opt (keys %$conf) {
5480         next if $opt !~ m/^unused\d+$/;
5481         my $volid = $conf->{$opt};
5482         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5483         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5484             $changes = 1;
5485             delete $conf->{$opt};
5486         }
5487
5488         $referenced->{$volid} = 1;
5489         $referencedpath->{$path} = 1 if $path;
5490     }
5491
5492     foreach my $volid (sort keys %$volid_hash) {
5493         next if $volid =~ m/vm-$vmid-state-/;
5494         next if $referenced->{$volid};
5495         my $path = $volid_hash->{$volid}->{path};
5496         next if !$path; # just to be sure
5497         next if $referencedpath->{$path};
5498         $changes = 1;
5499         PVE::QemuConfig->add_unused_volume($conf, $volid);
5500         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5501     }
5502
5503     return $changes;
5504 }
5505
5506 sub rescan {
5507     my ($vmid, $nolock) = @_;
5508
5509     my $cfg = PVE::Storage::config();
5510
5511     my $volid_hash = scan_volids($cfg, $vmid);
5512
5513     my $updatefn =  sub {
5514         my ($vmid) = @_;
5515
5516         my $conf = PVE::QemuConfig->load_config($vmid);
5517
5518         PVE::QemuConfig->check_lock($conf);
5519
5520         my $vm_volids = {};
5521         foreach my $volid (keys %$volid_hash) {
5522             my $info = $volid_hash->{$volid};
5523             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5524         }
5525
5526         my $changes = update_disksize($vmid, $conf, $vm_volids);
5527
5528         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5529     };
5530
5531     if (defined($vmid)) {
5532         if ($nolock) {
5533             &$updatefn($vmid);
5534         } else {
5535             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5536         }
5537     } else {
5538         my $vmlist = config_list();
5539         foreach my $vmid (keys %$vmlist) {
5540             if ($nolock) {
5541                 &$updatefn($vmid);
5542             } else {
5543                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5544             }
5545         }
5546     }
5547 }
5548
5549 sub restore_vma_archive {
5550     my ($archive, $vmid, $user, $opts, $comp) = @_;
5551
5552     my $input = $archive eq '-' ? "<&STDIN" : undef;
5553     my $readfrom = $archive;
5554
5555     my $uncomp = '';
5556     if ($comp) {
5557         $readfrom = '-';
5558         my $qarchive = PVE::Tools::shellquote($archive);
5559         if ($comp eq 'gzip') {
5560             $uncomp = "zcat $qarchive|";
5561         } elsif ($comp eq 'lzop') {
5562             $uncomp = "lzop -d -c $qarchive|";
5563         } else {
5564             die "unknown compression method '$comp'\n";
5565         }
5566
5567     }
5568
5569     my $tmpdir = "/var/tmp/vzdumptmp$$";
5570     rmtree $tmpdir;
5571
5572     # disable interrupts (always do cleanups)
5573     local $SIG{INT} =
5574         local $SIG{TERM} =
5575         local $SIG{QUIT} =
5576         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5577
5578     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5579     POSIX::mkfifo($mapfifo, 0600);
5580     my $fifofh;
5581
5582     my $openfifo = sub {
5583         open($fifofh, '>', $mapfifo) || die $!;
5584     };
5585
5586     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5587
5588     my $oldtimeout;
5589     my $timeout = 5;
5590
5591     my $devinfo = {};
5592
5593     my $rpcenv = PVE::RPCEnvironment::get();
5594
5595     my $conffile = PVE::QemuConfig->config_file($vmid);
5596     my $tmpfn = "$conffile.$$.tmp";
5597
5598     # Note: $oldconf is undef if VM does not exists
5599     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5600     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5601
5602     my $print_devmap = sub {
5603         my $virtdev_hash = {};
5604
5605         my $cfgfn = "$tmpdir/qemu-server.conf";
5606
5607         # we can read the config - that is already extracted
5608         my $fh = IO::File->new($cfgfn, "r") ||
5609             "unable to read qemu-server.conf - $!\n";
5610
5611         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5612         if (-f $fwcfgfn) {
5613             my $pve_firewall_dir = '/etc/pve/firewall';
5614             mkdir $pve_firewall_dir; # make sure the dir exists
5615             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5616         }
5617
5618         while (defined(my $line = <$fh>)) {
5619             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5620                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5621                 die "archive does not contain data for drive '$virtdev'\n"
5622                     if !$devinfo->{$devname};
5623                 if (defined($opts->{storage})) {
5624                     $storeid = $opts->{storage} || 'local';
5625                 } elsif (!$storeid) {
5626                     $storeid = 'local';
5627                 }
5628                 $format = 'raw' if !$format;
5629                 $devinfo->{$devname}->{devname} = $devname;
5630                 $devinfo->{$devname}->{virtdev} = $virtdev;
5631                 $devinfo->{$devname}->{format} = $format;
5632                 $devinfo->{$devname}->{storeid} = $storeid;
5633
5634                 # check permission on storage
5635                 my $pool = $opts->{pool}; # todo: do we need that?
5636                 if ($user ne 'root@pam') {
5637                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5638                 }
5639
5640                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5641             }
5642         }
5643
5644         foreach my $devname (keys %$devinfo) {
5645             die "found no device mapping information for device '$devname'\n"
5646                 if !$devinfo->{$devname}->{virtdev};
5647         }
5648
5649         my $cfg = PVE::Storage::config();
5650
5651         # create empty/temp config
5652         if ($oldconf) {
5653             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5654             foreach_drive($oldconf, sub {
5655                 my ($ds, $drive) = @_;
5656
5657                 return if drive_is_cdrom($drive);
5658
5659                 my $volid = $drive->{file};
5660
5661                 return if !$volid || $volid =~ m|^/|;
5662
5663                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5664                 return if !$path || !$owner || ($owner != $vmid);
5665
5666                 # Note: only delete disk we want to restore
5667                 # other volumes will become unused
5668                 if ($virtdev_hash->{$ds}) {
5669                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5670                     if (my $err = $@) {
5671                         warn $err;
5672                     }
5673                 }
5674             });
5675
5676             # delete vmstate files
5677             # since after the restore we have no snapshots anymore
5678             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5679                 my $snap = $oldconf->{snapshots}->{$snapname};
5680                 if ($snap->{vmstate}) {
5681                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5682                     if (my $err = $@) {
5683                         warn $err;
5684                     }
5685                 }
5686             }
5687         }
5688
5689         my $map = {};
5690         foreach my $virtdev (sort keys %$virtdev_hash) {
5691             my $d = $virtdev_hash->{$virtdev};
5692             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5693             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5694
5695             # test if requested format is supported
5696             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5697             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5698             $d->{format} = $defFormat if !$supported;
5699
5700             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5701                                                   $d->{format}, undef, $alloc_size);
5702             print STDERR "new volume ID is '$volid'\n";
5703             $d->{volid} = $volid;
5704             my $path = PVE::Storage::path($cfg, $volid);
5705
5706             PVE::Storage::activate_volumes($cfg,[$volid]);
5707
5708             my $write_zeros = 1;
5709             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5710                 $write_zeros = 0;
5711             }
5712
5713             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5714
5715             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5716             $map->{$virtdev} = $volid;
5717         }
5718
5719         $fh->seek(0, 0) || die "seek failed - $!\n";
5720
5721         my $outfd = new IO::File ($tmpfn, "w") ||
5722             die "unable to write config for VM $vmid\n";
5723
5724         my $cookie = { netcount => 0 };
5725         while (defined(my $line = <$fh>)) {
5726             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5727         }
5728
5729         $fh->close();
5730         $outfd->close();
5731     };
5732
5733     eval {
5734         # enable interrupts
5735         local $SIG{INT} =
5736             local $SIG{TERM} =
5737             local $SIG{QUIT} =
5738             local $SIG{HUP} =
5739             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5740         local $SIG{ALRM} = sub { die "got timeout\n"; };
5741
5742         $oldtimeout = alarm($timeout);
5743
5744         my $parser = sub {
5745             my $line = shift;
5746
5747             print "$line\n";
5748
5749             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5750                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5751                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5752             } elsif ($line =~ m/^CTIME: /) {
5753                 # we correctly received the vma config, so we can disable
5754                 # the timeout now for disk allocation (set to 10 minutes, so
5755                 # that we always timeout if something goes wrong)
5756                 alarm(600);
5757                 &$print_devmap();
5758                 print $fifofh "done\n";
5759                 my $tmp = $oldtimeout || 0;
5760                 $oldtimeout = undef;
5761                 alarm($tmp);
5762                 close($fifofh);
5763             }
5764         };
5765
5766         print "restore vma archive: $cmd\n";
5767         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5768     };
5769     my $err = $@;
5770
5771     alarm($oldtimeout) if $oldtimeout;
5772
5773     my $vollist = [];
5774     foreach my $devname (keys %$devinfo) {
5775         my $volid = $devinfo->{$devname}->{volid};
5776         push @$vollist, $volid if $volid;
5777     }
5778
5779     my $cfg = PVE::Storage::config();
5780     PVE::Storage::deactivate_volumes($cfg, $vollist);
5781
5782     unlink $mapfifo;
5783
5784     if ($err) {
5785         rmtree $tmpdir;
5786         unlink $tmpfn;
5787
5788         foreach my $devname (keys %$devinfo) {
5789             my $volid = $devinfo->{$devname}->{volid};
5790             next if !$volid;
5791             eval {
5792                 if ($volid =~ m|^/|) {
5793                     unlink $volid || die 'unlink failed\n';
5794                 } else {
5795                     PVE::Storage::vdisk_free($cfg, $volid);
5796                 }
5797                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5798             };
5799             print STDERR "unable to cleanup '$volid' - $@" if $@;
5800         }
5801         die $err;
5802     }
5803
5804     rmtree $tmpdir;
5805
5806     rename($tmpfn, $conffile) ||
5807         die "unable to commit configuration file '$conffile'\n";
5808
5809     PVE::Cluster::cfs_update(); # make sure we read new file
5810
5811     eval { rescan($vmid, 1); };
5812     warn $@ if $@;
5813 }
5814
5815 sub restore_tar_archive {
5816     my ($archive, $vmid, $user, $opts) = @_;
5817
5818     if ($archive ne '-') {
5819         my $firstfile = tar_archive_read_firstfile($archive);
5820         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5821             if $firstfile ne 'qemu-server.conf';
5822     }
5823
5824     my $storecfg = PVE::Storage::config();
5825
5826     # destroy existing data - keep empty config
5827     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5828     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5829
5830     my $tocmd = "/usr/lib/qemu-server/qmextract";
5831
5832     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5833     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5834     $tocmd .= ' --prealloc' if $opts->{prealloc};
5835     $tocmd .= ' --info' if $opts->{info};
5836
5837     # tar option "xf" does not autodetect compression when read from STDIN,
5838     # so we pipe to zcat
5839     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5840         PVE::Tools::shellquote("--to-command=$tocmd");
5841
5842     my $tmpdir = "/var/tmp/vzdumptmp$$";
5843     mkpath $tmpdir;
5844
5845     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5846     local $ENV{VZDUMP_VMID} = $vmid;
5847     local $ENV{VZDUMP_USER} = $user;
5848
5849     my $conffile = PVE::QemuConfig->config_file($vmid);
5850     my $tmpfn = "$conffile.$$.tmp";
5851
5852     # disable interrupts (always do cleanups)
5853     local $SIG{INT} =
5854         local $SIG{TERM} =
5855         local $SIG{QUIT} =
5856         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5857
5858     eval {
5859         # enable interrupts
5860         local $SIG{INT} =
5861             local $SIG{TERM} =
5862             local $SIG{QUIT} =
5863             local $SIG{HUP} =
5864             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5865
5866         if ($archive eq '-') {
5867             print "extracting archive from STDIN\n";
5868             run_command($cmd, input => "<&STDIN");
5869         } else {
5870             print "extracting archive '$archive'\n";
5871             run_command($cmd);
5872         }
5873
5874         return if $opts->{info};
5875
5876         # read new mapping
5877         my $map = {};
5878         my $statfile = "$tmpdir/qmrestore.stat";
5879         if (my $fd = IO::File->new($statfile, "r")) {
5880             while (defined (my $line = <$fd>)) {
5881                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5882                     $map->{$1} = $2 if $1;
5883                 } else {
5884                     print STDERR "unable to parse line in statfile - $line\n";
5885                 }
5886             }
5887             $fd->close();
5888         }
5889
5890         my $confsrc = "$tmpdir/qemu-server.conf";
5891
5892         my $srcfd = new IO::File($confsrc, "r") ||
5893             die "unable to open file '$confsrc'\n";
5894
5895         my $outfd = new IO::File ($tmpfn, "w") ||
5896             die "unable to write config for VM $vmid\n";
5897
5898         my $cookie = { netcount => 0 };
5899         while (defined (my $line = <$srcfd>)) {
5900             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5901         }
5902
5903         $srcfd->close();
5904         $outfd->close();
5905     };
5906     my $err = $@;
5907
5908     if ($err) {
5909
5910         unlink $tmpfn;
5911
5912         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5913
5914         die $err;
5915     }
5916
5917     rmtree $tmpdir;
5918
5919     rename $tmpfn, $conffile ||
5920         die "unable to commit configuration file '$conffile'\n";
5921
5922     PVE::Cluster::cfs_update(); # make sure we read new file
5923
5924     eval { rescan($vmid, 1); };
5925     warn $@ if $@;
5926 };
5927
5928 sub foreach_storage_used_by_vm {
5929     my ($conf, $func) = @_;
5930
5931     my $sidhash = {};
5932
5933     foreach_drive($conf, sub {
5934         my ($ds, $drive) = @_;
5935         return if drive_is_cdrom($drive);
5936
5937         my $volid = $drive->{file};
5938
5939         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5940         $sidhash->{$sid} = $sid if $sid;
5941     });
5942
5943     foreach my $sid (sort keys %$sidhash) {
5944         &$func($sid);
5945     }
5946 }
5947
5948 sub do_snapshots_with_qemu {
5949     my ($storecfg, $volid) = @_;
5950
5951     my $storage_name = PVE::Storage::parse_volume_id($volid);
5952
5953     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5954         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5955         return 1;
5956     }
5957
5958     if ($volid =~ m/\.(qcow2|qed)$/){
5959         return 1;
5960     }
5961
5962     return undef;
5963 }
5964
5965 sub qga_check_running {
5966     my ($vmid) = @_;
5967
5968     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5969     if ($@) {
5970         warn "Qemu Guest Agent is not running - $@";
5971         return 0;
5972     }
5973     return 1;
5974 }
5975
5976 sub template_create {
5977     my ($vmid, $conf, $disk) = @_;
5978
5979     my $storecfg = PVE::Storage::config();
5980
5981     foreach_drive($conf, sub {
5982         my ($ds, $drive) = @_;
5983
5984         return if drive_is_cdrom($drive);
5985         return if $disk && $ds ne $disk;
5986
5987         my $volid = $drive->{file};
5988         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5989
5990         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5991         $drive->{file} = $voliddst;
5992         $conf->{$ds} = print_drive($vmid, $drive);
5993         PVE::QemuConfig->write_config($vmid, $conf);
5994     });
5995 }
5996
5997 sub qemu_img_convert {
5998     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5999
6000     my $storecfg = PVE::Storage::config();
6001     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6002     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6003
6004     if ($src_storeid && $dst_storeid) {
6005
6006         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6007
6008         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6009         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6010
6011         my $src_format = qemu_img_format($src_scfg, $src_volname);
6012         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6013
6014         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6015         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6016
6017         my $cmd = [];
6018         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6019         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6020         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6021         if ($is_zero_initialized) {
6022             push @$cmd, "zeroinit:$dst_path";
6023         } else {
6024             push @$cmd, $dst_path;
6025         }
6026
6027         my $parser = sub {
6028             my $line = shift;
6029             if($line =~ m/\((\S+)\/100\%\)/){
6030                 my $percent = $1;
6031                 my $transferred = int($size * $percent / 100);
6032                 my $remaining = $size - $transferred;
6033
6034                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6035             }
6036
6037         };
6038
6039         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6040         my $err = $@;
6041         die "copy failed: $err" if $err;
6042     }
6043 }
6044
6045 sub qemu_img_format {
6046     my ($scfg, $volname) = @_;
6047
6048     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6049         return $1;
6050     } else {
6051         return "raw";
6052     }
6053 }
6054
6055 sub qemu_drive_mirror {
6056     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6057
6058     $jobs = {} if !$jobs;
6059
6060     my $qemu_target;
6061     my $format;
6062     $jobs->{"drive-$drive"} = {};
6063
6064     if ($dst_volid =~ /^nbd:/) {
6065         $qemu_target = $dst_volid;
6066         $format = "nbd";
6067     } else {
6068         my $storecfg = PVE::Storage::config();
6069         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6070
6071         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6072
6073         $format = qemu_img_format($dst_scfg, $dst_volname);
6074
6075         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6076
6077         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6078     }
6079
6080     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6081     $opts->{format} = $format if $format;
6082
6083     print "drive mirror is starting for drive-$drive\n";
6084
6085     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6086
6087     if (my $err = $@) {
6088         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6089         die "mirroring error: $err";
6090     }
6091
6092     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6093 }
6094
6095 sub qemu_drive_mirror_monitor {
6096     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6097
6098     eval {
6099         my $err_complete = 0;
6100
6101         while (1) {
6102             die "storage migration timed out\n" if $err_complete > 300;
6103
6104             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6105
6106             my $running_mirror_jobs = {};
6107             foreach my $stat (@$stats) {
6108                 next if $stat->{type} ne 'mirror';
6109                 $running_mirror_jobs->{$stat->{device}} = $stat;
6110             }
6111
6112             my $readycounter = 0;
6113
6114             foreach my $job (keys %$jobs) {
6115
6116                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6117                     print "$job : finished\n";
6118                     delete $jobs->{$job};
6119                     next;
6120                 }
6121
6122                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6123
6124                 my $busy = $running_mirror_jobs->{$job}->{busy};
6125                 my $ready = $running_mirror_jobs->{$job}->{ready};
6126                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6127                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6128                     my $remaining = $total - $transferred;
6129                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6130
6131                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6132                 }
6133
6134                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6135             }
6136
6137             last if scalar(keys %$jobs) == 0;
6138
6139             if ($readycounter == scalar(keys %$jobs)) {
6140                 print "all mirroring jobs are ready \n";
6141                 last if $skipcomplete; #do the complete later
6142
6143                 if ($vmiddst && $vmiddst != $vmid) {
6144                     my $agent_running = $qga && qga_check_running($vmid);
6145                     if ($agent_running) {
6146                         print "freeze filesystem\n";
6147                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6148                     } else {
6149                         print "suspend vm\n";
6150                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6151                     }
6152
6153                     # if we clone a disk for a new target vm, we don't switch the disk
6154                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6155
6156                     if ($agent_running) {
6157                         print "unfreeze filesystem\n";
6158                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6159                     } else {
6160                         print "resume vm\n";
6161                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6162                     }
6163
6164                     last;
6165                 } else {
6166
6167                     foreach my $job (keys %$jobs) {
6168                         # try to switch the disk if source and destination are on the same guest
6169                         print "$job: Completing block job...\n";
6170
6171                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6172                         if ($@ =~ m/cannot be completed/) {
6173                             print "$job: Block job cannot be completed, try again.\n";
6174                             $err_complete++;
6175                         }else {
6176                             print "$job: Completed successfully.\n";
6177                             $jobs->{$job}->{complete} = 1;
6178                         }
6179                     }
6180                 }
6181             }
6182             sleep 1;
6183         }
6184     };
6185     my $err = $@;
6186
6187     if ($err) {
6188         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6189         die "mirroring error: $err";
6190     }
6191
6192 }
6193
6194 sub qemu_blockjobs_cancel {
6195     my ($vmid, $jobs) = @_;
6196
6197     foreach my $job (keys %$jobs) {
6198         print "$job: Cancelling block job\n";
6199         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6200         $jobs->{$job}->{cancel} = 1;
6201     }
6202
6203     while (1) {
6204         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6205
6206         my $running_jobs = {};
6207         foreach my $stat (@$stats) {
6208             $running_jobs->{$stat->{device}} = $stat;
6209         }
6210
6211         foreach my $job (keys %$jobs) {
6212
6213             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6214                 print "$job: Done.\n";
6215                 delete $jobs->{$job};
6216             }
6217         }
6218
6219         last if scalar(keys %$jobs) == 0;
6220
6221         sleep 1;
6222     }
6223 }
6224
6225 sub clone_disk {
6226     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6227         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6228
6229     my $newvolid;
6230
6231     if (!$full) {
6232         print "create linked clone of drive $drivename ($drive->{file})\n";
6233         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6234         push @$newvollist, $newvolid;
6235     } else {
6236
6237         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6238         $storeid = $storage if $storage;
6239
6240         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6241         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6242
6243         print "create full clone of drive $drivename ($drive->{file})\n";
6244         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, undef, ($size/1024));
6245         push @$newvollist, $newvolid;
6246
6247         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6248
6249         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6250         if (!$running || $snapname) {
6251             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6252         } else {
6253
6254             my $kvmver = get_running_qemu_version ($vmid);
6255             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6256                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6257                     if $drive->{iothread};
6258             }
6259
6260             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6261         }
6262     }
6263
6264     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6265
6266     my $disk = $drive;
6267     $disk->{format} = undef;
6268     $disk->{file} = $newvolid;
6269     $disk->{size} = $size;
6270
6271     return $disk;
6272 }
6273
6274 # this only works if VM is running
6275 sub get_current_qemu_machine {
6276     my ($vmid) = @_;
6277
6278     my $cmd = { execute => 'query-machines', arguments => {} };
6279     my $res = vm_qmp_command($vmid, $cmd);
6280
6281     my ($current, $default);
6282     foreach my $e (@$res) {
6283         $default = $e->{name} if $e->{'is-default'};
6284         $current = $e->{name} if $e->{'is-current'};
6285     }
6286
6287     # fallback to the default machine if current is not supported by qemu
6288     return $current || $default || 'pc';
6289 }
6290
6291 sub get_running_qemu_version {
6292     my ($vmid) = @_;
6293     my $cmd = { execute => 'query-version', arguments => {} };
6294     my $res = vm_qmp_command($vmid, $cmd);
6295     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6296 }
6297
6298 sub qemu_machine_feature_enabled {
6299     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6300
6301     my $current_major;
6302     my $current_minor;
6303
6304     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6305
6306         $current_major = $3;
6307         $current_minor = $4;
6308
6309     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6310
6311         $current_major = $1;
6312         $current_minor = $2;
6313     }
6314
6315     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6316
6317
6318 }
6319
6320 sub qemu_machine_pxe {
6321     my ($vmid, $conf, $machine) = @_;
6322
6323     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6324
6325     foreach my $opt (keys %$conf) {
6326         next if $opt !~ m/^net(\d+)$/;
6327         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6328         next if !$net;
6329         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6330         return $machine.".pxe" if $romfile =~ m/pxe/;
6331         last;
6332     }
6333
6334     return $machine;
6335 }
6336
6337 sub qemu_use_old_bios_files {
6338     my ($machine_type) = @_;
6339
6340     return if !$machine_type;
6341
6342     my $use_old_bios_files = undef;
6343
6344     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6345         $machine_type = $1;
6346         $use_old_bios_files = 1;
6347     } else {
6348         my $kvmver = kvm_user_version();
6349         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6350         # load new efi bios files on migration. So this hack is required to allow
6351         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6352         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6353         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6354     }
6355
6356     return ($use_old_bios_files, $machine_type);
6357 }
6358
6359 sub create_efidisk {
6360     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6361
6362     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6363
6364     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6365     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6366     PVE::Storage::activate_volumes($storecfg, [$volid]);
6367
6368     my $path = PVE::Storage::path($storecfg, $volid);
6369     eval {
6370         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6371     };
6372     die "Copying EFI vars image failed: $@" if $@;
6373
6374     return ($volid, $vars_size);
6375 }
6376
6377 sub lspci {
6378
6379     my $devices = {};
6380
6381     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6382             my (undef, $id, $function) = @_;
6383             my $res = { id => $id, function => $function};
6384             push @{$devices->{$id}}, $res;
6385     });
6386
6387     # Entries should be sorted by functions.
6388     foreach my $id (keys %$devices) {
6389         my $dev = $devices->{$id};
6390         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6391     }
6392
6393     return $devices;
6394 }
6395
6396 sub vm_iothreads_list {
6397     my ($vmid) = @_;
6398
6399     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6400
6401     my $iothreads = {};
6402     foreach my $iothread (@$res) {
6403         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6404     }
6405
6406     return $iothreads;
6407 }
6408
6409 sub scsihw_infos {
6410     my ($conf, $drive) = @_;
6411
6412     my $maxdev = 0;
6413
6414     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6415         $maxdev = 7;
6416     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6417         $maxdev = 1;
6418     } else {
6419         $maxdev = 256;
6420     }
6421
6422     my $controller = int($drive->{index} / $maxdev);
6423     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6424
6425     return ($maxdev, $controller, $controller_prefix);
6426 }
6427
6428 sub add_hyperv_enlightenments {
6429     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6430
6431     return if $winversion < 6;
6432     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6433
6434     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6435
6436     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6437         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6438         push @$cpuFlags , 'hv_vapic';
6439         push @$cpuFlags , 'hv_time';
6440     } else {
6441         push @$cpuFlags , 'hv_spinlocks=0xffff';
6442     }
6443
6444     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6445         push @$cpuFlags , 'hv_reset';
6446         push @$cpuFlags , 'hv_vpindex';
6447         push @$cpuFlags , 'hv_runtime';
6448     }
6449
6450     if ($winversion >= 7) {
6451         push @$cpuFlags , 'hv_relaxed';
6452     }
6453 }
6454
6455 sub windows_version {
6456     my ($ostype) = @_;
6457
6458     return 0 if !$ostype;
6459
6460     my $winversion = 0;
6461
6462     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6463         $winversion = 5;
6464     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6465         $winversion = 6;
6466     } elsif ($ostype =~ m/^win(\d+)$/) {
6467         $winversion = $1;
6468     }
6469
6470     return $winversion;
6471 }
6472
6473 sub resolve_dst_disk_format {
6474         my ($storecfg, $storeid, $src_volname, $format) = @_;
6475         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6476
6477         if (!$format) {
6478             # if no target format is specified, use the source disk format as hint
6479             if ($src_volname) {
6480                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6481                 $format = qemu_img_format($scfg, $src_volname);
6482             } else {
6483                 return $defFormat;
6484             }
6485         }
6486
6487         # test if requested format is supported - else use default
6488         my $supported = grep { $_ eq $format } @$validFormats;
6489         $format = $defFormat if !$supported;
6490         return $format;
6491 }
6492
6493 sub resolve_first_disk {
6494     my $conf = shift;
6495     my @disks = PVE::QemuServer::valid_drive_names();
6496     my $firstdisk;
6497     foreach my $ds (reverse @disks) {
6498         next if !$conf->{$ds};
6499         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6500         next if PVE::QemuServer::drive_is_cdrom($disk);
6501         $firstdisk = $ds;
6502     }
6503     return $firstdisk;
6504 }
6505
6506 sub generate_smbios1_uuid {
6507     my ($uuid, $uuid_str);
6508     UUID::generate($uuid);
6509     UUID::unparse($uuid, $uuid_str);
6510     return "uuid=$uuid_str";
6511 }
6512
6513 # bash completion helper
6514
6515 sub complete_backup_archives {
6516     my ($cmdname, $pname, $cvalue) = @_;
6517
6518     my $cfg = PVE::Storage::config();
6519
6520     my $storeid;
6521
6522     if ($cvalue =~ m/^([^:]+):/) {
6523         $storeid = $1;
6524     }
6525
6526     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6527
6528     my $res = [];
6529     foreach my $id (keys %$data) {
6530         foreach my $item (@{$data->{$id}}) {
6531             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6532             push @$res, $item->{volid} if defined($item->{volid});
6533         }
6534     }
6535
6536     return $res;
6537 }
6538
6539 my $complete_vmid_full = sub {
6540     my ($running) = @_;
6541
6542     my $idlist = vmstatus();
6543
6544     my $res = [];
6545
6546     foreach my $id (keys %$idlist) {
6547         my $d = $idlist->{$id};
6548         if (defined($running)) {
6549             next if $d->{template};
6550             next if $running && $d->{status} ne 'running';
6551             next if !$running && $d->{status} eq 'running';
6552         }
6553         push @$res, $id;
6554
6555     }
6556     return $res;
6557 };
6558
6559 sub complete_vmid {
6560     return &$complete_vmid_full();
6561 }
6562
6563 sub complete_vmid_stopped {
6564     return &$complete_vmid_full(0);
6565 }
6566
6567 sub complete_vmid_running {
6568     return &$complete_vmid_full(1);
6569 }
6570
6571 sub complete_storage {
6572
6573     my $cfg = PVE::Storage::config();
6574     my $ids = $cfg->{ids};
6575
6576     my $res = [];
6577     foreach my $sid (keys %$ids) {
6578         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6579         next if !$ids->{$sid}->{content}->{images};
6580         push @$res, $sid;
6581     }
6582
6583     return $res;
6584 }
6585
6586 sub nbd_stop {
6587     my ($vmid) = @_;
6588
6589     vm_mon_cmd($vmid, 'nbd-server-stop');
6590 }
6591
6592 1;