PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42
  43 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  44
  45 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  46
  47 # Note about locking: we use flock on the config file protect
  48 # against concurent actions.
  49 # Aditionaly, we have a 'lock' setting in the config file. This
  50 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  51 # allowed when such lock is set. But you can ignore this kind of
  52 # lock with the --skiplock flag.
  53
  54 cfs_register_file('/qemu-server/',
  55                   \&parse_vm_config,
  56                   \&write_vm_config);
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  72     type => 'string',
  73     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  74     description => "The drive's backing file's data format.",
  75     optional => 1,
  76 });
  77
  78 #no warnings 'redefine';
  79
  80 sub cgroups_write {
  81    my ($controller, $vmid, $option, $value) = @_;
  82
  83    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  84    PVE::ProcFSTools::write_proc_entry($path, $value);
  85
  86 }
  87
  88 my $nodename = PVE::INotify::nodename();
  89
  90 mkdir "/etc/pve/nodes/$nodename";
  91 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  92 mkdir $confdir;
  93
  94 my $var_run_tmpdir = "/var/run/qemu-server";
  95 mkdir $var_run_tmpdir;
  96
  97 my $lock_dir = "/var/lock/qemu-server";
  98 mkdir $lock_dir;
  99
 100 my $pcisysfs = "/sys/bus/pci";
 101
 102 my $cpu_vendor_list = {
 103     # Intel CPUs
 104     486 => 'GenuineIntel',
 105     pentium => 'GenuineIntel',
 106     pentium2  => 'GenuineIntel',
 107     pentium3  => 'GenuineIntel',
 108     coreduo => 'GenuineIntel',
 109     core2duo => 'GenuineIntel',
 110     Conroe  => 'GenuineIntel',
 111     Penryn  => 'GenuineIntel',
 112     Nehalem  => 'GenuineIntel',
 113     'Nehalem-IBRS'  => 'GenuineIntel',
 114     Westmere => 'GenuineIntel',
 115     'Westmere-IBRS' => 'GenuineIntel',
 116     SandyBridge => 'GenuineIntel',
 117     'SandyBridge-IBRS' => 'GenuineIntel',
 118     IvyBridge => 'GenuineIntel',
 119     'IvyBridge-IBRS' => 'GenuineIntel',
 120     Haswell => 'GenuineIntel',
 121     'Haswell-IBRS' => 'GenuineIntel',
 122     'Haswell-noTSX' => 'GenuineIntel',
 123     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 124     Broadwell => 'GenuineIntel',
 125     'Broadwell-IBRS' => 'GenuineIntel',
 126     'Broadwell-noTSX' => 'GenuineIntel',
 127     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 128     'Skylake-Client' => 'GenuineIntel',
 129     'Skylake-Client-IBRS' => 'GenuineIntel',
 130     'Skylake-Server' => 'GenuineIntel',
 131     'Skylake-Server-IBRS' => 'GenuineIntel',
 132
 133     # AMD CPUs
 134     athlon => 'AuthenticAMD',
 135     phenom  => 'AuthenticAMD',
 136     Opteron_G1  => 'AuthenticAMD',
 137     Opteron_G2  => 'AuthenticAMD',
 138     Opteron_G3  => 'AuthenticAMD',
 139     Opteron_G4  => 'AuthenticAMD',
 140     Opteron_G5  => 'AuthenticAMD',
 141     EPYC => 'AuthenticAMD',
 142     'EPYC-IBPB' => 'AuthenticAMD',
 143
 144     # generic types, use vendor from host node
 145     host => 'default',
 146     kvm32 => 'default',
 147     kvm64 => 'default',
 148     qemu32 => 'default',
 149     qemu64 => 'default',
 150     max => 'default',
 151 };
 152
 153 my $cpu_flag = qr/[+-](pcid|spec-ctrl)/;
 154
 155 my $cpu_fmt = {
 156     cputype => {
 157         description => "Emulated CPU type.",
 158         type => 'string',
 159         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 160         default => 'kvm64',
 161         default_key => 1,
 162     },
 163     hidden => {
 164         description => "Do not identify as a KVM virtual machine.",
 165         type => 'boolean',
 166         optional => 1,
 167         default => 0
 168     },
 169     flags => {
 170         description => "List of additional CPU flags separated by ';'."
 171                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 172                      . " Currently supported flags: 'pcid', 'spec-ctrl'.",
 173         format_description => '+FLAG[;-FLAG...]',
 174         type => 'string',
 175         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 176         optional => 1,
 177     },
 178 };
 179
 180 my $watchdog_fmt = {
 181     model => {
 182         default_key => 1,
 183         type => 'string',
 184         enum => [qw(i6300esb ib700)],
 185         description => "Watchdog type to emulate.",
 186         default => 'i6300esb',
 187         optional => 1,
 188     },
 189     action => {
 190         type => 'string',
 191         enum => [qw(reset shutdown poweroff pause debug none)],
 192         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 193         optional => 1,
 194     },
 195 };
 196 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 197
 198 my $confdesc = {
 199     onboot => {
 200         optional => 1,
 201         type => 'boolean',
 202         description => "Specifies whether a VM will be started during system bootup.",
 203         default => 0,
 204     },
 205     autostart => {
 206         optional => 1,
 207         type => 'boolean',
 208         description => "Automatic restart after crash (currently ignored).",
 209         default => 0,
 210     },
 211     hotplug => {
 212         optional => 1,
 213         type => 'string', format => 'pve-hotplug-features',
 214         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 215         default => 'network,disk,usb',
 216     },
 217     reboot => {
 218         optional => 1,
 219         type => 'boolean',
 220         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 221         default => 1,
 222     },
 223     lock => {
 224         optional => 1,
 225         type => 'string',
 226         description => "Lock/unlock the VM.",
 227         enum => [qw(migrate backup snapshot rollback)],
 228     },
 229     cpulimit => {
 230         optional => 1,
 231         type => 'number',
 232         description => "Limit of CPU usage.",
 233         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 234         minimum => 0,
 235         maximum => 128,
 236         default => 0,
 237     },
 238     cpuunits => {
 239         optional => 1,
 240         type => 'integer',
 241         description => "CPU weight for a VM.",
 242         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 243         minimum => 2,
 244         maximum => 262144,
 245         default => 1024,
 246     },
 247     memory => {
 248         optional => 1,
 249         type => 'integer',
 250         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 251         minimum => 16,
 252         default => 512,
 253     },
 254     balloon => {
 255         optional => 1,
 256         type => 'integer',
 257         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 258         minimum => 0,
 259     },
 260     shares => {
 261         optional => 1,
 262         type => 'integer',
 263         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 264         minimum => 0,
 265         maximum => 50000,
 266         default => 1000,
 267     },
 268     keyboard => {
 269         optional => 1,
 270         type => 'string',
 271         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.".
 272                        "It should not be necessary to set it.",
 273         enum => PVE::Tools::kvmkeymaplist(),
 274         default => undef,
 275     },
 276     name => {
 277         optional => 1,
 278         type => 'string', format => 'dns-name',
 279         description => "Set a name for the VM. Only used on the configuration web interface.",
 280     },
 281     scsihw => {
 282         optional => 1,
 283         type => 'string',
 284         description => "SCSI controller model",
 285         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 286         default => 'lsi',
 287     },
 288     description => {
 289         optional => 1,
 290         type => 'string',
 291         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 292     },
 293     ostype => {
 294         optional => 1,
 295         type => 'string',
 296         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 297         description => "Specify guest operating system.",
 298         verbose_description => <<EODESC,
 299 Specify guest operating system. This is used to enable special
 300 optimization/features for specific operating systems:
 301
 302 [horizontal]
 303 other;; unspecified OS
 304 wxp;; Microsoft Windows XP
 305 w2k;; Microsoft Windows 2000
 306 w2k3;; Microsoft Windows 2003
 307 w2k8;; Microsoft Windows 2008
 308 wvista;; Microsoft Windows Vista
 309 win7;; Microsoft Windows 7
 310 win8;; Microsoft Windows 8/2012/2012r2
 311 win10;; Microsoft Windows 10/2016
 312 l24;; Linux 2.4 Kernel
 313 l26;; Linux 2.6/3.X Kernel
 314 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 315 EODESC
 316     },
 317     boot => {
 318         optional => 1,
 319         type => 'string',
 320         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 321         pattern => '[acdn]{1,4}',
 322         default => 'cdn',
 323     },
 324     bootdisk => {
 325         optional => 1,
 326         type => 'string', format => 'pve-qm-bootdisk',
 327         description => "Enable booting from specified disk.",
 328         pattern => '(ide|sata|scsi|virtio)\d+',
 329     },
 330     smp => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "The number of CPUs. Please use option -sockets instead.",
 334         minimum => 1,
 335         default => 1,
 336     },
 337     sockets => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "The number of CPU sockets.",
 341         minimum => 1,
 342         default => 1,
 343     },
 344     cores => {
 345         optional => 1,
 346         type => 'integer',
 347         description => "The number of cores per socket.",
 348         minimum => 1,
 349         default => 1,
 350     },
 351     numa => {
 352         optional => 1,
 353         type => 'boolean',
 354         description => "Enable/disable NUMA.",
 355         default => 0,
 356     },
 357     hugepages => {
 358         optional => 1,
 359         type => 'string',
 360         description => "Enable/disable hugepages memory.",
 361         enum => [qw(any 2 1024)],
 362     },
 363     vcpus => {
 364         optional => 1,
 365         type => 'integer',
 366         description => "Number of hotplugged vcpus.",
 367         minimum => 1,
 368         default => 0,
 369     },
 370     acpi => {
 371         optional => 1,
 372         type => 'boolean',
 373         description => "Enable/disable ACPI.",
 374         default => 1,
 375     },
 376     agent => {
 377         optional => 1,
 378         type => 'boolean',
 379         description => "Enable/disable Qemu GuestAgent.",
 380         default => 0,
 381     },
 382     kvm => {
 383         optional => 1,
 384         type => 'boolean',
 385         description => "Enable/disable KVM hardware virtualization.",
 386         default => 1,
 387     },
 388     tdf => {
 389         optional => 1,
 390         type => 'boolean',
 391         description => "Enable/disable time drift fix.",
 392         default => 0,
 393     },
 394     localtime => {
 395         optional => 1,
 396         type => 'boolean',
 397         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 398     },
 399     freeze => {
 400         optional => 1,
 401         type => 'boolean',
 402         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 403     },
 404     vga => {
 405         optional => 1,
 406         type => 'string',
 407         description => "Select the VGA type.",
 408         verbose_description => "Select the VGA type. If you want to use high resolution" .
 409             " modes (>= 1280x1024x16) then you should use the options " .
 410             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 411             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 412             "display sever. For win* OS you can select how many independent " .
 413             "displays you want, Linux guests can add displays them self. " .
 414             "You can also run without any graphic card, using a serial device" .
 415             " as terminal.",
 416         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 417     },
 418     watchdog => {
 419         optional => 1,
 420         type => 'string', format => 'pve-qm-watchdog',
 421         description => "Create a virtual hardware watchdog device.",
 422         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 423             " (by a guest action), the watchdog must be periodically polled " .
 424             "by an agent inside the guest or else the watchdog will reset " .
 425             "the guest (or execute the respective action specified)",
 426     },
 427     startdate => {
 428         optional => 1,
 429         type => 'string',
 430         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 431         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 432         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 433         default => 'now',
 434     },
 435     startup =>  get_standard_option('pve-startup-order'),
 436     template => {
 437         optional => 1,
 438         type => 'boolean',
 439         description => "Enable/disable Template.",
 440         default => 0,
 441     },
 442     args => {
 443         optional => 1,
 444         type => 'string',
 445         description => "Arbitrary arguments passed to kvm.",
 446         verbose_description => <<EODESCR,
 447 Arbitrary arguments passed to kvm, for example:
 448
 449 args: -no-reboot -no-hpet
 450
 451 NOTE: this option is for experts only.
 452 EODESCR
 453     },
 454     tablet => {
 455         optional => 1,
 456         type => 'boolean',
 457         default => 1,
 458         description => "Enable/disable the USB tablet device.",
 459         verbose_description => "Enable/disable the USB tablet device. This device is " .
 460             "usually needed to allow absolute mouse positioning with VNC. " .
 461             "Else the mouse runs out of sync with normal VNC clients. " .
 462             "If you're running lots of console-only guests on one host, " .
 463             "you may consider disabling this to save some context switches. " .
 464             "This is turned off by default if you use spice (-vga=qxl).",
 465     },
 466     migrate_speed => {
 467         optional => 1,
 468         type => 'integer',
 469         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 470         minimum => 0,
 471         default => 0,
 472     },
 473     migrate_downtime => {
 474         optional => 1,
 475         type => 'number',
 476         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 477         minimum => 0,
 478         default => 0.1,
 479     },
 480     cdrom => {
 481         optional => 1,
 482         type => 'string', format => 'pve-qm-ide',
 483         typetext => '<volume>',
 484         description => "This is an alias for option -ide2",
 485     },
 486     cpu => {
 487         optional => 1,
 488         description => "Emulated CPU type.",
 489         type => 'string',
 490         format => $cpu_fmt,
 491     },
 492     parent => get_standard_option('pve-snapshot-name', {
 493         optional => 1,
 494         description => "Parent snapshot name. This is used internally, and should not be modified.",
 495     }),
 496     snaptime => {
 497         optional => 1,
 498         description => "Timestamp for snapshots.",
 499         type => 'integer',
 500         minimum => 0,
 501     },
 502     vmstate => {
 503         optional => 1,
 504         type => 'string', format => 'pve-volume-id',
 505         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 506     },
 507     vmstatestorage => get_standard_option('pve-storage-id', {
 508         description => "Default storage for VM state volumes/files.",
 509         optional => 1,
 510     }),
 511     machine => {
 512         description => "Specific the Qemu machine type.",
 513         type => 'string',
 514         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 515         maxLength => 40,
 516         optional => 1,
 517     },
 518     smbios1 => {
 519         description => "Specify SMBIOS type 1 fields.",
 520         type => 'string', format => 'pve-qm-smbios1',
 521         maxLength => 256,
 522         optional => 1,
 523     },
 524     protection => {
 525         optional => 1,
 526         type => 'boolean',
 527         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 528         default => 0,
 529     },
 530     bios => {
 531         optional => 1,
 532         type => 'string',
 533         enum => [ qw(seabios ovmf) ],
 534         description => "Select BIOS implementation.",
 535         default => 'seabios',
 536     },
 537 };
 538
 539 # what about other qemu settings ?
 540 #cpu => 'string',
 541 #machine => 'string',
 542 #fda => 'file',
 543 #fdb => 'file',
 544 #mtdblock => 'file',
 545 #sd => 'file',
 546 #pflash => 'file',
 547 #snapshot => 'bool',
 548 #bootp => 'file',
 549 ##tftp => 'dir',
 550 ##smb => 'dir',
 551 #kernel => 'file',
 552 #append => 'string',
 553 #initrd => 'file',
 554 ##soundhw => 'string',
 555
 556 while (my ($k, $v) = each %$confdesc) {
 557     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 558 }
 559
 560 my $MAX_IDE_DISKS = 4;
 561 my $MAX_SCSI_DISKS = 14;
 562 my $MAX_VIRTIO_DISKS = 16;
 563 my $MAX_SATA_DISKS = 6;
 564 my $MAX_USB_DEVICES = 5;
 565 my $MAX_NETS = 32;
 566 my $MAX_UNUSED_DISKS = 8;
 567 my $MAX_HOSTPCI_DEVICES = 4;
 568 my $MAX_SERIAL_PORTS = 4;
 569 my $MAX_PARALLEL_PORTS = 3;
 570 my $MAX_NUMA = 8;
 571
 572 my $numa_fmt = {
 573     cpus => {
 574         type => "string",
 575         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 576         description => "CPUs accessing this NUMA node.",
 577         format_description => "id[-id];...",
 578     },
 579     memory => {
 580         type => "number",
 581         description => "Amount of memory this NUMA node provides.",
 582         optional => 1,
 583     },
 584     hostnodes => {
 585         type => "string",
 586         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 587         description => "Host NUMA nodes to use.",
 588         format_description => "id[-id];...",
 589         optional => 1,
 590     },
 591     policy => {
 592         type => 'string',
 593         enum => [qw(preferred bind interleave)],
 594         description => "NUMA allocation policy.",
 595         optional => 1,
 596     },
 597 };
 598 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 599 my $numadesc = {
 600     optional => 1,
 601     type => 'string', format => $numa_fmt,
 602     description => "NUMA topology.",
 603 };
 604 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 605
 606 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 607     $confdesc->{"numa$i"} = $numadesc;
 608 }
 609
 610 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 611                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 612                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 613 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 614
 615 my $net_fmt_bridge_descr = <<__EOD__;
 616 Bridge to attach the network device to. The Proxmox VE standard bridge
 617 is called 'vmbr0'.
 618
 619 If you do not specify a bridge, we create a kvm user (NATed) network
 620 device, which provides DHCP and DNS services. The following addresses
 621 are used:
 622
 623  10.0.2.2   Gateway
 624  10.0.2.3   DNS Server
 625  10.0.2.4   SMB Server
 626
 627 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 628 __EOD__
 629
 630 my $net_fmt = {
 631     macaddr => {
 632         type => 'string',
 633         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 634         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 635         format_description => "XX:XX:XX:XX:XX:XX",
 636         optional => 1,
 637     },
 638     model => {
 639         type => 'string',
 640         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 641         enum => $nic_model_list,
 642         default_key => 1,
 643     },
 644     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 645     bridge => {
 646         type => 'string',
 647         description => $net_fmt_bridge_descr,
 648         format_description => 'bridge',
 649         optional => 1,
 650     },
 651     queues => {
 652         type => 'integer',
 653         minimum => 0, maximum => 16,
 654         description => 'Number of packet queues to be used on the device.',
 655         optional => 1,
 656     },
 657     rate => {
 658         type => 'number',
 659         minimum => 0,
 660         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 661         optional => 1,
 662     },
 663     tag => {
 664         type => 'integer',
 665         minimum => 1, maximum => 4094,
 666         description => 'VLAN tag to apply to packets on this interface.',
 667         optional => 1,
 668     },
 669     trunks => {
 670         type => 'string',
 671         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 672         description => 'VLAN trunks to pass through this interface.',
 673         format_description => 'vlanid[;vlanid...]',
 674         optional => 1,
 675     },
 676     firewall => {
 677         type => 'boolean',
 678         description => 'Whether this interface should be protected by the firewall.',
 679         optional => 1,
 680     },
 681     link_down => {
 682         type => 'boolean',
 683         description => 'Whether this interface should be disconnected (like pulling the plug).',
 684         optional => 1,
 685     },
 686 };
 687
 688 my $netdesc = {
 689     optional => 1,
 690     type => 'string', format => $net_fmt,
 691     description => "Specify network devices.",
 692 };
 693
 694 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 695
 696 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 697     $confdesc->{"net$i"} = $netdesc;
 698 }
 699
 700 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 701 sub verify_volume_id_or_qm_path {
 702     my ($volid, $noerr) = @_;
 703
 704     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 705         return $volid;
 706     }
 707
 708     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 709     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 710     if ($@) {
 711         return undef if $noerr;
 712         die $@;
 713     }
 714     return $volid;
 715 }
 716
 717 my $drivename_hash;
 718
 719 my %drivedesc_base = (
 720     volume => { alias => 'file' },
 721     file => {
 722         type => 'string',
 723         format => 'pve-volume-id-or-qm-path',
 724         default_key => 1,
 725         format_description => 'volume',
 726         description => "The drive's backing volume.",
 727     },
 728     media => {
 729         type => 'string',
 730         enum => [qw(cdrom disk)],
 731         description => "The drive's media type.",
 732         default => 'disk',
 733         optional => 1
 734     },
 735     cyls => {
 736         type => 'integer',
 737         description => "Force the drive's physical geometry to have a specific cylinder count.",
 738         optional => 1
 739     },
 740     heads => {
 741         type => 'integer',
 742         description => "Force the drive's physical geometry to have a specific head count.",
 743         optional => 1
 744     },
 745     secs => {
 746         type => 'integer',
 747         description => "Force the drive's physical geometry to have a specific sector count.",
 748         optional => 1
 749     },
 750     trans => {
 751         type => 'string',
 752         enum => [qw(none lba auto)],
 753         description => "Force disk geometry bios translation mode.",
 754         optional => 1,
 755     },
 756     snapshot => {
 757         type => 'boolean',
 758         description => "Controls qemu's snapshot mode feature."
 759             . " If activated, changes made to the disk are temporary and will"
 760             . " be discarded when the VM is shutdown.",
 761         optional => 1,
 762     },
 763     cache => {
 764         type => 'string',
 765         enum => [qw(none writethrough writeback unsafe directsync)],
 766         description => "The drive's cache mode",
 767         optional => 1,
 768     },
 769     format => get_standard_option('pve-qm-image-format'),
 770     size => {
 771         type => 'string',
 772         format => 'disk-size',
 773         format_description => 'DiskSize',
 774         description => "Disk size. This is purely informational and has no effect.",
 775         optional => 1,
 776     },
 777     backup => {
 778         type => 'boolean',
 779         description => "Whether the drive should be included when making backups.",
 780         optional => 1,
 781     },
 782     replicate => {
 783         type => 'boolean',
 784         description => 'Whether the drive should considered for replication jobs.',
 785         optional => 1,
 786         default => 1,
 787     },
 788     rerror => {
 789         type => 'string',
 790         enum => [qw(ignore report stop)],
 791         description => 'Read error action.',
 792         optional => 1,
 793     },
 794     werror => {
 795         type => 'string',
 796         enum => [qw(enospc ignore report stop)],
 797         description => 'Write error action.',
 798         optional => 1,
 799     },
 800     aio => {
 801         type => 'string',
 802         enum => [qw(native threads)],
 803         description => 'AIO type to use.',
 804         optional => 1,
 805     },
 806     discard => {
 807         type => 'string',
 808         enum => [qw(ignore on)],
 809         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 810         optional => 1,
 811     },
 812     detect_zeroes => {
 813         type => 'boolean',
 814         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 815         optional => 1,
 816     },
 817     serial => {
 818         type => 'string',
 819         format => 'urlencoded',
 820         format_description => 'serial',
 821         maxLength => 20*3, # *3 since it's %xx url enoded
 822         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 823         optional => 1,
 824     },
 825     shared => {
 826         type => 'boolean',
 827         description => 'Mark this locally-managed volume as available on all nodes',
 828         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 829         optional => 1,
 830         default => 0,
 831     }
 832 );
 833
 834 my %iothread_fmt = ( iothread => {
 835         type => 'boolean',
 836         description => "Whether to use iothreads for this drive",
 837         optional => 1,
 838 });
 839
 840 my %model_fmt = (
 841     model => {
 842         type => 'string',
 843         format => 'urlencoded',
 844         format_description => 'model',
 845         maxLength => 40*3, # *3 since it's %xx url enoded
 846         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 847         optional => 1,
 848     },
 849 );
 850
 851 my %queues_fmt = (
 852     queues => {
 853         type => 'integer',
 854         description => "Number of queues.",
 855         minimum => 2,
 856         optional => 1
 857     }
 858 );
 859
 860 my %scsiblock_fmt = (
 861     scsiblock => {
 862         type => 'boolean',
 863         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 864         optional => 1,
 865         default => 0,
 866     },
 867 );
 868
 869 my $add_throttle_desc = sub {
 870     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 871     my $d = {
 872         type => $type,
 873         format_description => $unit,
 874         description => "Maximum $what in $longunit.",
 875         optional => 1,
 876     };
 877     $d->{minimum} = $minimum if defined($minimum);
 878     $drivedesc_base{$key} = $d;
 879 };
 880 # throughput: (leaky bucket)
 881 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 882 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 883 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 884 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 885 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 886 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 887 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 888 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 889 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 890
 891 # pools: (pool of IO before throttling starts taking effect)
 892 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 893 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 894 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 895 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 896 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 897 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 898
 899 # burst lengths
 900 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 901 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 902 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 903 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 904 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 905 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 906
 907 # legacy support
 908 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
 909 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
 910 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
 911 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
 912
 913 my $ide_fmt = {
 914     %drivedesc_base,
 915     %model_fmt,
 916 };
 917 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 918
 919 my $idedesc = {
 920     optional => 1,
 921     type => 'string', format => $ide_fmt,
 922     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 923 };
 924 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 925
 926 my $scsi_fmt = {
 927     %drivedesc_base,
 928     %iothread_fmt,
 929     %queues_fmt,
 930     %scsiblock_fmt,
 931 };
 932 my $scsidesc = {
 933     optional => 1,
 934     type => 'string', format => $scsi_fmt,
 935     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 936 };
 937 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 938
 939 my $sata_fmt = {
 940     %drivedesc_base,
 941 };
 942 my $satadesc = {
 943     optional => 1,
 944     type => 'string', format => $sata_fmt,
 945     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 946 };
 947 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 948
 949 my $virtio_fmt = {
 950     %drivedesc_base,
 951     %iothread_fmt,
 952 };
 953 my $virtiodesc = {
 954     optional => 1,
 955     type => 'string', format => $virtio_fmt,
 956     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 957 };
 958 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 959
 960 my $alldrive_fmt = {
 961     %drivedesc_base,
 962     %iothread_fmt,
 963     %model_fmt,
 964     %queues_fmt,
 965     %scsiblock_fmt,
 966 };
 967
 968 my $efidisk_fmt = {
 969     volume => { alias => 'file' },
 970     file => {
 971         type => 'string',
 972         format => 'pve-volume-id-or-qm-path',
 973         default_key => 1,
 974         format_description => 'volume',
 975         description => "The drive's backing volume.",
 976     },
 977     format => get_standard_option('pve-qm-image-format'),
 978     size => {
 979         type => 'string',
 980         format => 'disk-size',
 981         format_description => 'DiskSize',
 982         description => "Disk size. This is purely informational and has no effect.",
 983         optional => 1,
 984     },
 985 };
 986
 987 my $efidisk_desc = {
 988     optional => 1,
 989     type => 'string', format => $efidisk_fmt,
 990     description => "Configure a Disk for storing EFI vars",
 991 };
 992
 993 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 994
 995 my $usb_fmt = {
 996     host => {
 997         default_key => 1,
 998         type => 'string', format => 'pve-qm-usb-device',
 999         format_description => 'HOSTUSBDEVICE|spice',
1000         description => <<EODESCR,
1001 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1002
1003  'bus-port(.port)*' (decimal numbers) or
1004  'vendor_id:product_id' (hexadeciaml numbers) or
1005  'spice'
1006
1007 You can use the 'lsusb -t' command to list existing usb devices.
1008
1009 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1010
1011 The value 'spice' can be used to add a usb redirection devices for spice.
1012 EODESCR
1013     },
1014     usb3 => {
1015         optional => 1,
1016         type => 'boolean',
1017         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1018         default => 0,
1019     },
1020 };
1021
1022 my $usbdesc = {
1023     optional => 1,
1024     type => 'string', format => $usb_fmt,
1025     description => "Configure an USB device (n is 0 to 4).",
1026 };
1027 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1028
1029 # NOTE: the match-groups of this regex are used in parse_hostpci
1030 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1031 my $hostpci_fmt = {
1032     host => {
1033         default_key => 1,
1034         type => 'string',
1035         pattern => qr/$PCIRE(;$PCIRE)*/,
1036         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1037         description => <<EODESCR,
1038 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1039 of PCI virtual functions of the host. HOSTPCIID syntax is:
1040
1041 'bus:dev.func' (hexadecimal numbers)
1042
1043 You can us the 'lspci' command to list existing PCI devices.
1044 EODESCR
1045     },
1046     rombar => {
1047         type => 'boolean',
1048         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1049         optional => 1,
1050         default => 1,
1051     },
1052     romfile => {
1053         type => 'string',
1054         pattern => '[^,;]+',
1055         format_description => 'string',
1056         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1057         optional => 1,
1058     },
1059     pcie => {
1060         type => 'boolean',
1061         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1062         optional => 1,
1063         default => 0,
1064     },
1065     'x-vga' => {
1066         type => 'boolean',
1067         description =>  "Enable vfio-vga device support.",
1068         optional => 1,
1069         default => 0,
1070     },
1071 };
1072 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1073
1074 my $hostpcidesc = {
1075         optional => 1,
1076         type => 'string', format => 'pve-qm-hostpci',
1077         description => "Map host PCI devices into guest.",
1078         verbose_description =>  <<EODESCR,
1079 Map host PCI devices into guest.
1080
1081 NOTE: This option allows direct access to host hardware. So it is no longer
1082 possible to migrate such machines - use with special care.
1083
1084 CAUTION: Experimental! User reported problems with this option.
1085 EODESCR
1086 };
1087 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1088
1089 my $serialdesc = {
1090         optional => 1,
1091         type => 'string',
1092         pattern => '(/dev/.+|socket)',
1093         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1094         verbose_description =>  <<EODESCR,
1095 Create a serial device inside the VM (n is 0 to 3), and pass through a
1096 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1097 host side (use 'qm terminal' to open a terminal connection).
1098
1099 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1100
1101 CAUTION: Experimental! User reported problems with this option.
1102 EODESCR
1103 };
1104
1105 my $paralleldesc= {
1106         optional => 1,
1107         type => 'string',
1108         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1109         description =>  "Map host parallel devices (n is 0 to 2).",
1110         verbose_description =>  <<EODESCR,
1111 Map host parallel devices (n is 0 to 2).
1112
1113 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1114
1115 CAUTION: Experimental! User reported problems with this option.
1116 EODESCR
1117 };
1118
1119 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1120     $confdesc->{"parallel$i"} = $paralleldesc;
1121 }
1122
1123 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1124     $confdesc->{"serial$i"} = $serialdesc;
1125 }
1126
1127 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1128     $confdesc->{"hostpci$i"} = $hostpcidesc;
1129 }
1130
1131 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1132     $drivename_hash->{"ide$i"} = 1;
1133     $confdesc->{"ide$i"} = $idedesc;
1134 }
1135
1136 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1137     $drivename_hash->{"sata$i"} = 1;
1138     $confdesc->{"sata$i"} = $satadesc;
1139 }
1140
1141 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1142     $drivename_hash->{"scsi$i"} = 1;
1143     $confdesc->{"scsi$i"} = $scsidesc ;
1144 }
1145
1146 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1147     $drivename_hash->{"virtio$i"} = 1;
1148     $confdesc->{"virtio$i"} = $virtiodesc;
1149 }
1150
1151 $drivename_hash->{efidisk0} = 1;
1152 $confdesc->{efidisk0} = $efidisk_desc;
1153
1154 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1155     $confdesc->{"usb$i"} = $usbdesc;
1156 }
1157
1158 my $unuseddesc = {
1159     optional => 1,
1160     type => 'string', format => 'pve-volume-id',
1161     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1162 };
1163
1164 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1165     $confdesc->{"unused$i"} = $unuseddesc;
1166 }
1167
1168 my $kvm_api_version = 0;
1169
1170 sub kvm_version {
1171
1172     return $kvm_api_version if $kvm_api_version;
1173
1174     my $fh = IO::File->new("</dev/kvm") ||
1175         return 0;
1176
1177     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1178         $kvm_api_version = $v;
1179     }
1180
1181     $fh->close();
1182
1183     return  $kvm_api_version;
1184 }
1185
1186 my $kvm_user_version;
1187
1188 sub kvm_user_version {
1189
1190     return $kvm_user_version if $kvm_user_version;
1191
1192     $kvm_user_version = 'unknown';
1193
1194     my $code = sub {
1195         my $line = shift;
1196         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1197             $kvm_user_version = $2;
1198         }
1199     };
1200
1201     eval { run_command("kvm -version", outfunc => $code); };
1202     warn $@ if $@;
1203
1204     return $kvm_user_version;
1205
1206 }
1207
1208 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1209
1210 sub valid_drive_names {
1211     # order is important - used to autoselect boot disk
1212     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1213             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1214             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1215             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1216             'efidisk0');
1217 }
1218
1219 sub is_valid_drivename {
1220     my $dev = shift;
1221
1222     return defined($drivename_hash->{$dev});
1223 }
1224
1225 sub option_exists {
1226     my $key = shift;
1227     return defined($confdesc->{$key});
1228 }
1229
1230 sub nic_models {
1231     return $nic_model_list;
1232 }
1233
1234 sub os_list_description {
1235
1236     return {
1237         other => 'Other',
1238         wxp => 'Windows XP',
1239         w2k => 'Windows 2000',
1240         w2k3 =>, 'Windows 2003',
1241         w2k8 => 'Windows 2008',
1242         wvista => 'Windows Vista',
1243         win7 => 'Windows 7',
1244         win8 => 'Windows 8/2012',
1245         win10 => 'Windows 10/2016',
1246         l24 => 'Linux 2.4',
1247         l26 => 'Linux 2.6',
1248     };
1249 }
1250
1251 my $cdrom_path;
1252
1253 sub get_cdrom_path {
1254
1255     return  $cdrom_path if $cdrom_path;
1256
1257     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1258     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1259     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1260 }
1261
1262 sub get_iso_path {
1263     my ($storecfg, $vmid, $cdrom) = @_;
1264
1265     if ($cdrom eq 'cdrom') {
1266         return get_cdrom_path();
1267     } elsif ($cdrom eq 'none') {
1268         return '';
1269     } elsif ($cdrom =~ m|^/|) {
1270         return $cdrom;
1271     } else {
1272         return PVE::Storage::path($storecfg, $cdrom);
1273     }
1274 }
1275
1276 # try to convert old style file names to volume IDs
1277 sub filename_to_volume_id {
1278     my ($vmid, $file, $media) = @_;
1279
1280     if (!($file eq 'none' || $file eq 'cdrom' ||
1281           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1282
1283         return undef if $file =~ m|/|;
1284
1285         if ($media && $media eq 'cdrom') {
1286             $file = "local:iso/$file";
1287         } else {
1288             $file = "local:$vmid/$file";
1289         }
1290     }
1291
1292     return $file;
1293 }
1294
1295 sub verify_media_type {
1296     my ($opt, $vtype, $media) = @_;
1297
1298     return if !$media;
1299
1300     my $etype;
1301     if ($media eq 'disk') {
1302         $etype = 'images';
1303     } elsif ($media eq 'cdrom') {
1304         $etype = 'iso';
1305     } else {
1306         die "internal error";
1307     }
1308
1309     return if ($vtype eq $etype);
1310
1311     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1312 }
1313
1314 sub cleanup_drive_path {
1315     my ($opt, $storecfg, $drive) = @_;
1316
1317     # try to convert filesystem paths to volume IDs
1318
1319     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1320         ($drive->{file} !~ m|^/dev/.+|) &&
1321         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1322         ($drive->{file} !~ m/^\d+$/)) {
1323         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1324         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1325         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1326         verify_media_type($opt, $vtype, $drive->{media});
1327         $drive->{file} = $volid;
1328     }
1329
1330     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1331 }
1332
1333 sub parse_hotplug_features {
1334     my ($data) = @_;
1335
1336     my $res = {};
1337
1338     return $res if $data eq '0';
1339
1340     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1341
1342     foreach my $feature (PVE::Tools::split_list($data)) {
1343         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1344             $res->{$1} = 1;
1345         } else {
1346             die "invalid hotplug feature '$feature'\n";
1347         }
1348     }
1349     return $res;
1350 }
1351
1352 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1353 sub pve_verify_hotplug_features {
1354     my ($value, $noerr) = @_;
1355
1356     return $value if parse_hotplug_features($value);
1357
1358     return undef if $noerr;
1359
1360     die "unable to parse hotplug option\n";
1361 }
1362
1363 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1364 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1365 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1366 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1367 #        [,iothread=on][,serial=serial][,model=model]
1368
1369 sub parse_drive {
1370     my ($key, $data) = @_;
1371
1372     my ($interface, $index);
1373
1374     if ($key =~ m/^([^\d]+)(\d+)$/) {
1375         $interface = $1;
1376         $index = $2;
1377     } else {
1378         return undef;
1379     }
1380
1381     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1382                                      : $confdesc->{$key}->{format};
1383     if (!$desc) {
1384         warn "invalid drive key: $key\n";
1385         return undef;
1386     }
1387     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1388     return undef if !$res;
1389     $res->{interface} = $interface;
1390     $res->{index} = $index;
1391
1392     my $error = 0;
1393     foreach my $opt (qw(bps bps_rd bps_wr)) {
1394         if (my $bps = defined(delete $res->{$opt})) {
1395             if (defined($res->{"m$opt"})) {
1396                 warn "both $opt and m$opt specified\n";
1397                 ++$error;
1398                 next;
1399             }
1400             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1401         }
1402     }
1403
1404     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1405     for my $requirement (
1406         [mbps_max => 'mbps'],
1407         [mbps_rd_max => 'mbps_rd'],
1408         [mbps_wr_max => 'mbps_wr'],
1409         [miops_max => 'miops'],
1410         [miops_rd_max => 'miops_rd'],
1411         [miops_wr_max => 'miops_wr'],
1412         [bps_max_length => 'mbps_max'],
1413         [bps_rd_max_length => 'mbps_rd_max'],
1414         [bps_wr_max_length => 'mbps_wr_max'],
1415         [iops_max_length => 'iops_max'],
1416         [iops_rd_max_length => 'iops_rd_max'],
1417         [iops_wr_max_length => 'iops_wr_max']) {
1418         my ($option, $requires) = @$requirement;
1419         if ($res->{$option} && !$res->{$requires}) {
1420             warn "$option requires $requires\n";
1421             ++$error;
1422         }
1423     }
1424
1425     return undef if $error;
1426
1427     return undef if $res->{mbps_rd} && $res->{mbps};
1428     return undef if $res->{mbps_wr} && $res->{mbps};
1429     return undef if $res->{iops_rd} && $res->{iops};
1430     return undef if $res->{iops_wr} && $res->{iops};
1431
1432     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1433         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1434         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1435         return undef if $res->{interface} eq 'virtio';
1436     }
1437
1438     if (my $size = $res->{size}) {
1439         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1440     }
1441
1442     return $res;
1443 }
1444
1445 sub print_drive {
1446     my ($vmid, $drive) = @_;
1447     my $data = { %$drive };
1448     delete $data->{$_} for qw(index interface);
1449     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1450 }
1451
1452 sub scsi_inquiry {
1453     my($fh, $noerr) = @_;
1454
1455     my $SG_IO = 0x2285;
1456     my $SG_GET_VERSION_NUM = 0x2282;
1457
1458     my $versionbuf = "\x00" x 8;
1459     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1460     if (!$ret) {
1461         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1462         return undef;
1463     }
1464     my $version = unpack("I", $versionbuf);
1465     if ($version < 30000) {
1466         die "scsi generic interface too old\n"  if !$noerr;
1467         return undef;
1468     }
1469
1470     my $buf = "\x00" x 36;
1471     my $sensebuf = "\x00" x 8;
1472     my $cmd = pack("C x3 C x1", 0x12, 36);
1473
1474     # see /usr/include/scsi/sg.h
1475     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1476
1477     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1478                       length($sensebuf), 0, length($buf), $buf,
1479                       $cmd, $sensebuf, 6000);
1480
1481     $ret = ioctl($fh, $SG_IO, $packet);
1482     if (!$ret) {
1483         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1484         return undef;
1485     }
1486
1487     my @res = unpack($sg_io_hdr_t, $packet);
1488     if ($res[17] || $res[18]) {
1489         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1490         return undef;
1491     }
1492
1493     my $res = {};
1494     (my $byte0, my $byte1, $res->{vendor},
1495      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1496
1497     $res->{removable} = $byte1 & 128 ? 1 : 0;
1498     $res->{type} = $byte0 & 31;
1499
1500     return $res;
1501 }
1502
1503 sub path_is_scsi {
1504     my ($path) = @_;
1505
1506     my $fh = IO::File->new("+<$path") || return undef;
1507     my $res = scsi_inquiry($fh, 1);
1508     close($fh);
1509
1510     return $res;
1511 }
1512
1513 sub machine_type_is_q35 {
1514     my ($conf) = @_;
1515
1516     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1517 }
1518
1519 sub print_tabletdevice_full {
1520     my ($conf) = @_;
1521
1522     my $q35 = machine_type_is_q35($conf);
1523
1524     # we use uhci for old VMs because tablet driver was buggy in older qemu
1525     my $usbbus = $q35 ? "ehci" : "uhci";
1526
1527     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1528 }
1529
1530 sub print_drivedevice_full {
1531     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1532
1533     my $device = '';
1534     my $maxdev = 0;
1535
1536     if ($drive->{interface} eq 'virtio') {
1537         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1538         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1539         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1540     } elsif ($drive->{interface} eq 'scsi') {
1541
1542         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1543         my $unit = $drive->{index} % $maxdev;
1544         my $devicetype = 'hd';
1545         my $path = '';
1546         if (drive_is_cdrom($drive)) {
1547             $devicetype = 'cd';
1548         } else {
1549             if ($drive->{file} =~ m|^/|) {
1550                 $path = $drive->{file};
1551                 if (my $info = path_is_scsi($path)) {
1552                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1553                         $devicetype = 'block';
1554                     } elsif ($info->{type} == 1) { # tape
1555                         $devicetype = 'generic';
1556                     }
1557                 }
1558             } else {
1559                  $path = PVE::Storage::path($storecfg, $drive->{file});
1560             }
1561
1562             if($path =~ m/^iscsi\:\/\//){
1563                 $devicetype = 'generic';
1564             }
1565         }
1566
1567         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1568             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1569         } else {
1570             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1571         }
1572
1573     } elsif ($drive->{interface} eq 'ide'){
1574         $maxdev = 2;
1575         my $controller = int($drive->{index} / $maxdev);
1576         my $unit = $drive->{index} % $maxdev;
1577         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1578
1579         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1580         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1581             $model = URI::Escape::uri_unescape($model);
1582             $device .= ",model=$model";
1583         }
1584     } elsif ($drive->{interface} eq 'sata'){
1585         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1586         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1587         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1588     } elsif ($drive->{interface} eq 'usb') {
1589         die "implement me";
1590         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1591     } else {
1592         die "unsupported interface type";
1593     }
1594
1595     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1596
1597     return $device;
1598 }
1599
1600 sub get_initiator_name {
1601     my $initiator;
1602
1603     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1604     while (defined(my $line = <$fh>)) {
1605         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1606         $initiator = $1;
1607         last;
1608     }
1609     $fh->close();
1610
1611     return $initiator;
1612 }
1613
1614 sub print_drive_full {
1615     my ($storecfg, $vmid, $drive) = @_;
1616
1617     my $path;
1618     my $volid = $drive->{file};
1619     my $format;
1620
1621     if (drive_is_cdrom($drive)) {
1622         $path = get_iso_path($storecfg, $vmid, $volid);
1623     } else {
1624         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1625         if ($storeid) {
1626             $path = PVE::Storage::path($storecfg, $volid);
1627             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1628             $format = qemu_img_format($scfg, $volname);
1629         } else {
1630             $path = $volid;
1631             $format = "raw";
1632         }
1633    }
1634
1635     my $opts = '';
1636     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1637     foreach my $o (@qemu_drive_options) {
1638         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1639     }
1640
1641     # snapshot only accepts on|off
1642     if (defined($drive->{snapshot})) {
1643         my $v = $drive->{snapshot} ? 'on' : 'off';
1644         $opts .= ",snapshot=$v";
1645     }
1646
1647     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1648         my ($dir, $qmpname) = @$type;
1649         if (my $v = $drive->{"mbps$dir"}) {
1650             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1651         }
1652         if (my $v = $drive->{"mbps${dir}_max"}) {
1653             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1654         }
1655         if (my $v = $drive->{"bps${dir}_max_length"}) {
1656             $opts .= ",throttling.bps$qmpname-max-length=$v";
1657         }
1658         if (my $v = $drive->{"iops${dir}"}) {
1659             $opts .= ",throttling.iops$qmpname=$v";
1660         }
1661         if (my $v = $drive->{"iops${dir}_max"}) {
1662             $opts .= ",throttling.iops$qmpname-max=$v";
1663         }
1664         if (my $v = $drive->{"iops${dir}_max_length"}) {
1665             $opts .= ",throttling.iops$qmpname-max-length=$v";
1666         }
1667     }
1668
1669     if (my $serial = $drive->{serial}) {
1670         $serial = URI::Escape::uri_unescape($serial);
1671         $opts .= ",serial=$serial";
1672     }
1673
1674     $opts .= ",format=$format" if $format && !$drive->{format};
1675
1676     my $cache_direct = 0;
1677
1678     if (my $cache = $drive->{cache}) {
1679         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1680     } elsif (!drive_is_cdrom($drive)) {
1681         $opts .= ",cache=none";
1682         $cache_direct = 1;
1683     }
1684
1685     # aio native works only with O_DIRECT
1686     if (!$drive->{aio}) {
1687         if($cache_direct) {
1688             $opts .= ",aio=native";
1689         } else {
1690             $opts .= ",aio=threads";
1691         }
1692     }
1693
1694     if (!drive_is_cdrom($drive)) {
1695         my $detectzeroes;
1696         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1697             $detectzeroes = 'off';
1698         } elsif ($drive->{discard}) {
1699             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1700         } else {
1701             # This used to be our default with discard not being specified:
1702             $detectzeroes = 'on';
1703         }
1704         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1705     }
1706
1707     my $pathinfo = $path ? "file=$path," : '';
1708
1709     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1710 }
1711
1712 sub print_netdevice_full {
1713     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1714
1715     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1716
1717     my $device = $net->{model};
1718     if ($net->{model} eq 'virtio') {
1719          $device = 'virtio-net-pci';
1720      };
1721
1722     my $pciaddr = print_pci_addr("$netid", $bridges);
1723     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1724     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1725         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1726         my $vectors = $net->{queues} * 2 + 2;
1727         $tmpstr .= ",vectors=$vectors,mq=on";
1728     }
1729     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1730
1731     if ($use_old_bios_files) {
1732         my $romfile;
1733         if ($device eq 'virtio-net-pci') {
1734             $romfile = 'pxe-virtio.rom';
1735         } elsif ($device eq 'e1000') {
1736             $romfile = 'pxe-e1000.rom';
1737         } elsif ($device eq 'ne2k') {
1738             $romfile = 'pxe-ne2k_pci.rom';
1739         } elsif ($device eq 'pcnet') {
1740             $romfile = 'pxe-pcnet.rom';
1741         } elsif ($device eq 'rtl8139') {
1742             $romfile = 'pxe-rtl8139.rom';
1743         }
1744         $tmpstr .= ",romfile=$romfile" if $romfile;
1745     }
1746
1747     return $tmpstr;
1748 }
1749
1750 sub print_netdev_full {
1751     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1752
1753     my $i = '';
1754     if ($netid =~ m/^net(\d+)$/) {
1755         $i = int($1);
1756     }
1757
1758     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1759
1760     my $ifname = "tap${vmid}i$i";
1761
1762     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1763     die "interface name '$ifname' is too long (max 15 character)\n"
1764         if length($ifname) >= 16;
1765
1766     my $vhostparam = '';
1767     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1768
1769     my $vmname = $conf->{name} || "vm$vmid";
1770
1771     my $netdev = "";
1772     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1773
1774     if ($net->{bridge}) {
1775         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1776     } else {
1777         $netdev = "type=user,id=$netid,hostname=$vmname";
1778     }
1779
1780     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1781
1782     return $netdev;
1783 }
1784
1785
1786 sub print_cpu_device {
1787     my ($conf, $id) = @_;
1788
1789     my $kvm = $conf->{kvm} // 1;
1790     my $cpu = $kvm ? "kvm64" : "qemu64";
1791     if (my $cputype = $conf->{cpu}) {
1792         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1793             or die "Cannot parse cpu description: $cputype\n";
1794         $cpu = $cpuconf->{cputype};
1795     }
1796
1797     my $cores = $conf->{cores} || 1;
1798
1799     my $current_core = ($id - 1) % $cores;
1800     my $current_socket = int(($id - 1 - $current_core)/$cores);
1801
1802     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1803 }
1804
1805 sub drive_is_cdrom {
1806     my ($drive) = @_;
1807
1808     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1809
1810 }
1811
1812 sub parse_number_sets {
1813     my ($set) = @_;
1814     my $res = [];
1815     foreach my $part (split(/;/, $set)) {
1816         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1817             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1818             push @$res, [ $1, $2 ];
1819         } else {
1820             die "invalid range: $part\n";
1821         }
1822     }
1823     return $res;
1824 }
1825
1826 sub parse_numa {
1827     my ($data) = @_;
1828
1829     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1830     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1831     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1832     return $res;
1833 }
1834
1835 sub parse_hostpci {
1836     my ($value) = @_;
1837
1838     return undef if !$value;
1839
1840     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1841
1842     my @idlist = split(/;/, $res->{host});
1843     delete $res->{host};
1844     foreach my $id (@idlist) {
1845         if ($id =~ /^$PCIRE$/) {
1846             if (defined($2)) {
1847                 push @{$res->{pciid}}, { id => $1, function => $2 };
1848             } else {
1849                 my $pcidevices = lspci($1);
1850                 $res->{pciid} = $pcidevices->{$1};
1851             }
1852         } else {
1853             # should have been caught by parse_property_string already
1854             die "failed to parse PCI id: $id\n";
1855         }
1856     }
1857     return $res;
1858 }
1859
1860 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1861 sub parse_net {
1862     my ($data) = @_;
1863
1864     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1865     if ($@) {
1866         warn $@;
1867         return undef;
1868     }
1869     if (!defined($res->{macaddr})) {
1870         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1871         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1872     }
1873     return $res;
1874 }
1875
1876 sub print_net {
1877     my $net = shift;
1878
1879     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1880 }
1881
1882 sub add_random_macs {
1883     my ($settings) = @_;
1884
1885     foreach my $opt (keys %$settings) {
1886         next if $opt !~ m/^net(\d+)$/;
1887         my $net = parse_net($settings->{$opt});
1888         next if !$net;
1889         $settings->{$opt} = print_net($net);
1890     }
1891 }
1892
1893 sub vm_is_volid_owner {
1894     my ($storecfg, $vmid, $volid) = @_;
1895
1896     if ($volid !~  m|^/|) {
1897         my ($path, $owner);
1898         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1899         if ($owner && ($owner == $vmid)) {
1900             return 1;
1901         }
1902     }
1903
1904     return undef;
1905 }
1906
1907 sub split_flagged_list {
1908     my $text = shift || '';
1909     $text =~ s/[,;]/ /g;
1910     $text =~ s/^\s+//;
1911     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1912 }
1913
1914 sub join_flagged_list {
1915     my ($how, $lst) = @_;
1916     join $how, map { $lst->{$_} . $_ } keys %$lst;
1917 }
1918
1919 sub vmconfig_delete_pending_option {
1920     my ($conf, $key, $force) = @_;
1921
1922     delete $conf->{pending}->{$key};
1923     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1924     $pending_delete_hash->{$key} = $force ? '!' : '';
1925     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1926 }
1927
1928 sub vmconfig_undelete_pending_option {
1929     my ($conf, $key) = @_;
1930
1931     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1932     delete $pending_delete_hash->{$key};
1933
1934     if (%$pending_delete_hash) {
1935         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1936     } else {
1937         delete $conf->{pending}->{delete};
1938     }
1939 }
1940
1941 sub vmconfig_register_unused_drive {
1942     my ($storecfg, $vmid, $conf, $drive) = @_;
1943
1944     if (!drive_is_cdrom($drive)) {
1945         my $volid = $drive->{file};
1946         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1947             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1948         }
1949     }
1950 }
1951
1952 sub vmconfig_cleanup_pending {
1953     my ($conf) = @_;
1954
1955     # remove pending changes when nothing changed
1956     my $changes;
1957     foreach my $opt (keys %{$conf->{pending}}) {
1958         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1959             $changes = 1;
1960             delete $conf->{pending}->{$opt};
1961         }
1962     }
1963
1964     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1965     my $pending_delete_hash = {};
1966     while (my ($opt, $force) = each %$current_delete_hash) {
1967         if (defined($conf->{$opt})) {
1968             $pending_delete_hash->{$opt} = $force;
1969         } else {
1970             $changes = 1;
1971         }
1972     }
1973
1974     if (%$pending_delete_hash) {
1975         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1976     } else {
1977         delete $conf->{pending}->{delete};
1978     }
1979
1980     return $changes;
1981 }
1982
1983 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1984 my $smbios1_fmt = {
1985     uuid => {
1986         type => 'string',
1987         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1988         format_description => 'UUID',
1989         description => "Set SMBIOS1 UUID.",
1990         optional => 1,
1991     },
1992     version => {
1993         type => 'string',
1994         pattern => '\S+',
1995         format_description => 'string',
1996         description => "Set SMBIOS1 version.",
1997         optional => 1,
1998     },
1999     serial => {
2000         type => 'string',
2001         pattern => '\S+',
2002         format_description => 'string',
2003         description => "Set SMBIOS1 serial number.",
2004         optional => 1,
2005     },
2006     manufacturer => {
2007         type => 'string',
2008         pattern => '\S+',
2009         format_description => 'string',
2010         description => "Set SMBIOS1 manufacturer.",
2011         optional => 1,
2012     },
2013     product => {
2014         type => 'string',
2015         pattern => '\S+',
2016         format_description => 'string',
2017         description => "Set SMBIOS1 product ID.",
2018         optional => 1,
2019     },
2020     sku => {
2021         type => 'string',
2022         pattern => '\S+',
2023         format_description => 'string',
2024         description => "Set SMBIOS1 SKU string.",
2025         optional => 1,
2026     },
2027     family => {
2028         type => 'string',
2029         pattern => '\S+',
2030         format_description => 'string',
2031         description => "Set SMBIOS1 family string.",
2032         optional => 1,
2033     },
2034 };
2035
2036 sub parse_smbios1 {
2037     my ($data) = @_;
2038
2039     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2040     warn $@ if $@;
2041     return $res;
2042 }
2043
2044 sub print_smbios1 {
2045     my ($smbios1) = @_;
2046     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2047 }
2048
2049 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2050
2051 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2052 sub verify_bootdisk {
2053     my ($value, $noerr) = @_;
2054
2055     return $value if is_valid_drivename($value);
2056
2057     return undef if $noerr;
2058
2059     die "invalid boot disk '$value'\n";
2060 }
2061
2062 sub parse_watchdog {
2063     my ($value) = @_;
2064
2065     return undef if !$value;
2066
2067     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2068     warn $@ if $@;
2069     return $res;
2070 }
2071
2072 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2073 sub verify_usb_device {
2074     my ($value, $noerr) = @_;
2075
2076     return $value if parse_usb_device($value);
2077
2078     return undef if $noerr;
2079
2080     die "unable to parse usb device\n";
2081 }
2082
2083 # add JSON properties for create and set function
2084 sub json_config_properties {
2085     my $prop = shift;
2086
2087     foreach my $opt (keys %$confdesc) {
2088         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2089         $prop->{$opt} = $confdesc->{$opt};
2090     }
2091
2092     return $prop;
2093 }
2094
2095 sub check_type {
2096     my ($key, $value) = @_;
2097
2098     die "unknown setting '$key'\n" if !$confdesc->{$key};
2099
2100     my $type = $confdesc->{$key}->{type};
2101
2102     if (!defined($value)) {
2103         die "got undefined value\n";
2104     }
2105
2106     if ($value =~ m/[\n\r]/) {
2107         die "property contains a line feed\n";
2108     }
2109
2110     if ($type eq 'boolean') {
2111         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2112         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2113         die "type check ('boolean') failed - got '$value'\n";
2114     } elsif ($type eq 'integer') {
2115         return int($1) if $value =~ m/^(\d+)$/;
2116         die "type check ('integer') failed - got '$value'\n";
2117     } elsif ($type eq 'number') {
2118         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2119         die "type check ('number') failed - got '$value'\n";
2120     } elsif ($type eq 'string') {
2121         if (my $fmt = $confdesc->{$key}->{format}) {
2122             PVE::JSONSchema::check_format($fmt, $value);
2123             return $value;
2124         }
2125         $value =~ s/^\"(.*)\"$/$1/;
2126         return $value;
2127     } else {
2128         die "internal error"
2129     }
2130 }
2131
2132 sub check_iommu_support{
2133     #fixme : need to check IOMMU support
2134     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2135
2136     my $iommu=1;
2137     return $iommu;
2138
2139 }
2140
2141 sub touch_config {
2142     my ($vmid) = @_;
2143
2144     my $conf = PVE::QemuConfig->config_file($vmid);
2145     utime undef, undef, $conf;
2146 }
2147
2148 sub destroy_vm {
2149     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2150
2151     my $conffile = PVE::QemuConfig->config_file($vmid);
2152
2153     my $conf = PVE::QemuConfig->load_config($vmid);
2154
2155     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2156
2157     if ($conf->{template}) {
2158         # check if any base image is still used by a linked clone
2159         foreach_drive($conf, sub {
2160                 my ($ds, $drive) = @_;
2161
2162                 return if drive_is_cdrom($drive);
2163
2164                 my $volid = $drive->{file};
2165
2166                 return if !$volid || $volid =~ m|^/|;
2167
2168                 die "base volume '$volid' is still in use by linked cloned\n"
2169                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2170
2171         });
2172     }
2173
2174     # only remove disks owned by this VM
2175     foreach_drive($conf, sub {
2176         my ($ds, $drive) = @_;
2177
2178         return if drive_is_cdrom($drive);
2179
2180         my $volid = $drive->{file};
2181
2182         return if !$volid || $volid =~ m|^/|;
2183
2184         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2185         return if !$path || !$owner || ($owner != $vmid);
2186
2187         eval {
2188             PVE::Storage::vdisk_free($storecfg, $volid);
2189         };
2190         warn "Could not remove disk '$volid', check manually: $@" if $@;
2191
2192     });
2193
2194     if ($keep_empty_config) {
2195         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2196     } else {
2197         unlink $conffile;
2198     }
2199
2200     # also remove unused disk
2201     eval {
2202         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2203
2204         eval {
2205             PVE::Storage::foreach_volid($dl, sub {
2206                 my ($volid, $sid, $volname, $d) = @_;
2207                 PVE::Storage::vdisk_free($storecfg, $volid);
2208             });
2209         };
2210         warn $@ if $@;
2211
2212     };
2213     warn $@ if $@;
2214 }
2215
2216 sub parse_vm_config {
2217     my ($filename, $raw) = @_;
2218
2219     return undef if !defined($raw);
2220
2221     my $res = {
2222         digest => Digest::SHA::sha1_hex($raw),
2223         snapshots => {},
2224         pending => {},
2225     };
2226
2227     $filename =~ m|/qemu-server/(\d+)\.conf$|
2228         || die "got strange filename '$filename'";
2229
2230     my $vmid = $1;
2231
2232     my $conf = $res;
2233     my $descr;
2234     my $section = '';
2235
2236     my @lines = split(/\n/, $raw);
2237     foreach my $line (@lines) {
2238         next if $line =~ m/^\s*$/;
2239
2240         if ($line =~ m/^\[PENDING\]\s*$/i) {
2241             $section = 'pending';
2242             if (defined($descr)) {
2243                 $descr =~ s/\s+$//;
2244                 $conf->{description} = $descr;
2245             }
2246             $descr = undef;
2247             $conf = $res->{$section} = {};
2248             next;
2249
2250         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2251             $section = $1;
2252             if (defined($descr)) {
2253                 $descr =~ s/\s+$//;
2254                 $conf->{description} = $descr;
2255             }
2256             $descr = undef;
2257             $conf = $res->{snapshots}->{$section} = {};
2258             next;
2259         }
2260
2261         if ($line =~ m/^\#(.*)\s*$/) {
2262             $descr = '' if !defined($descr);
2263             $descr .= PVE::Tools::decode_text($1) . "\n";
2264             next;
2265         }
2266
2267         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2268             $descr = '' if !defined($descr);
2269             $descr .= PVE::Tools::decode_text($2);
2270         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2271             $conf->{snapstate} = $1;
2272         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2273             my $key = $1;
2274             my $value = $2;
2275             $conf->{$key} = $value;
2276         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2277             my $value = $1;
2278             if ($section eq 'pending') {
2279                 $conf->{delete} = $value; # we parse this later
2280             } else {
2281                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2282             }
2283         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2284             my $key = $1;
2285             my $value = $2;
2286             eval { $value = check_type($key, $value); };
2287             if ($@) {
2288                 warn "vm $vmid - unable to parse value of '$key' - $@";
2289             } else {
2290                 $key = 'ide2' if $key eq 'cdrom';
2291                 my $fmt = $confdesc->{$key}->{format};
2292                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2293                     my $v = parse_drive($key, $value);
2294                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2295                         $v->{file} = $volid;
2296                         $value = print_drive($vmid, $v);
2297                     } else {
2298                         warn "vm $vmid - unable to parse value of '$key'\n";
2299                         next;
2300                     }
2301                 }
2302
2303                 $conf->{$key} = $value;
2304             }
2305         }
2306     }
2307
2308     if (defined($descr)) {
2309         $descr =~ s/\s+$//;
2310         $conf->{description} = $descr;
2311     }
2312     delete $res->{snapstate}; # just to be sure
2313
2314     return $res;
2315 }
2316
2317 sub write_vm_config {
2318     my ($filename, $conf) = @_;
2319
2320     delete $conf->{snapstate}; # just to be sure
2321
2322     if ($conf->{cdrom}) {
2323         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2324         $conf->{ide2} = $conf->{cdrom};
2325         delete $conf->{cdrom};
2326     }
2327
2328     # we do not use 'smp' any longer
2329     if ($conf->{sockets}) {
2330         delete $conf->{smp};
2331     } elsif ($conf->{smp}) {
2332         $conf->{sockets} = $conf->{smp};
2333         delete $conf->{cores};
2334         delete $conf->{smp};
2335     }
2336
2337     my $used_volids = {};
2338
2339     my $cleanup_config = sub {
2340         my ($cref, $pending, $snapname) = @_;
2341
2342         foreach my $key (keys %$cref) {
2343             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2344                 $key eq 'snapstate' || $key eq 'pending';
2345             my $value = $cref->{$key};
2346             if ($key eq 'delete') {
2347                 die "propertry 'delete' is only allowed in [PENDING]\n"
2348                     if !$pending;
2349                 # fixme: check syntax?
2350                 next;
2351             }
2352             eval { $value = check_type($key, $value); };
2353             die "unable to parse value of '$key' - $@" if $@;
2354
2355             $cref->{$key} = $value;
2356
2357             if (!$snapname && is_valid_drivename($key)) {
2358                 my $drive = parse_drive($key, $value);
2359                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2360             }
2361         }
2362     };
2363
2364     &$cleanup_config($conf);
2365
2366     &$cleanup_config($conf->{pending}, 1);
2367
2368     foreach my $snapname (keys %{$conf->{snapshots}}) {
2369         die "internal error" if $snapname eq 'pending';
2370         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2371     }
2372
2373     # remove 'unusedX' settings if we re-add a volume
2374     foreach my $key (keys %$conf) {
2375         my $value = $conf->{$key};
2376         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2377             delete $conf->{$key};
2378         }
2379     }
2380
2381     my $generate_raw_config = sub {
2382         my ($conf, $pending) = @_;
2383
2384         my $raw = '';
2385
2386         # add description as comment to top of file
2387         if (defined(my $descr = $conf->{description})) {
2388             if ($descr) {
2389                 foreach my $cl (split(/\n/, $descr)) {
2390                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2391                 }
2392             } else {
2393                 $raw .= "#\n" if $pending;
2394             }
2395         }
2396
2397         foreach my $key (sort keys %$conf) {
2398             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2399             $raw .= "$key: $conf->{$key}\n";
2400         }
2401         return $raw;
2402     };
2403
2404     my $raw = &$generate_raw_config($conf);
2405
2406     if (scalar(keys %{$conf->{pending}})){
2407         $raw .= "\n[PENDING]\n";
2408         $raw .= &$generate_raw_config($conf->{pending}, 1);
2409     }
2410
2411     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2412         $raw .= "\n[$snapname]\n";
2413         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2414     }
2415
2416     return $raw;
2417 }
2418
2419 sub load_defaults {
2420
2421     my $res = {};
2422
2423     # we use static defaults from our JSON schema configuration
2424     foreach my $key (keys %$confdesc) {
2425         if (defined(my $default = $confdesc->{$key}->{default})) {
2426             $res->{$key} = $default;
2427         }
2428     }
2429
2430     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2431     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2432
2433     return $res;
2434 }
2435
2436 sub config_list {
2437     my $vmlist = PVE::Cluster::get_vmlist();
2438     my $res = {};
2439     return $res if !$vmlist || !$vmlist->{ids};
2440     my $ids = $vmlist->{ids};
2441
2442     foreach my $vmid (keys %$ids) {
2443         my $d = $ids->{$vmid};
2444         next if !$d->{node} || $d->{node} ne $nodename;
2445         next if !$d->{type} || $d->{type} ne 'qemu';
2446         $res->{$vmid}->{exists} = 1;
2447     }
2448     return $res;
2449 }
2450
2451 # test if VM uses local resources (to prevent migration)
2452 sub check_local_resources {
2453     my ($conf, $noerr) = @_;
2454
2455     my $loc_res = 0;
2456
2457     $loc_res = 1 if $conf->{hostusb}; # old syntax
2458     $loc_res = 1 if $conf->{hostpci}; # old syntax
2459
2460     foreach my $k (keys %$conf) {
2461         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2462         # sockets are safe: they will recreated be on the target side post-migrate
2463         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2464         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2465     }
2466
2467     die "VM uses local resources\n" if $loc_res && !$noerr;
2468
2469     return $loc_res;
2470 }
2471
2472 # check if used storages are available on all nodes (use by migrate)
2473 sub check_storage_availability {
2474     my ($storecfg, $conf, $node) = @_;
2475
2476     foreach_drive($conf, sub {
2477         my ($ds, $drive) = @_;
2478
2479         my $volid = $drive->{file};
2480         return if !$volid;
2481
2482         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2483         return if !$sid;
2484
2485         # check if storage is available on both nodes
2486         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2487         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2488    });
2489 }
2490
2491 # list nodes where all VM images are available (used by has_feature API)
2492 sub shared_nodes {
2493     my ($conf, $storecfg) = @_;
2494
2495     my $nodelist = PVE::Cluster::get_nodelist();
2496     my $nodehash = { map { $_ => 1 } @$nodelist };
2497     my $nodename = PVE::INotify::nodename();
2498
2499     foreach_drive($conf, sub {
2500         my ($ds, $drive) = @_;
2501
2502         my $volid = $drive->{file};
2503         return if !$volid;
2504
2505         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2506         if ($storeid) {
2507             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2508             if ($scfg->{disable}) {
2509                 $nodehash = {};
2510             } elsif (my $avail = $scfg->{nodes}) {
2511                 foreach my $node (keys %$nodehash) {
2512                     delete $nodehash->{$node} if !$avail->{$node};
2513                 }
2514             } elsif (!$scfg->{shared}) {
2515                 foreach my $node (keys %$nodehash) {
2516                     delete $nodehash->{$node} if $node ne $nodename
2517                 }
2518             }
2519         }
2520     });
2521
2522     return $nodehash
2523 }
2524
2525 sub check_cmdline {
2526     my ($pidfile, $pid) = @_;
2527
2528     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2529     if (defined($fh)) {
2530         my $line = <$fh>;
2531         $fh->close;
2532         return undef if !$line;
2533         my @param = split(/\0/, $line);
2534
2535         my $cmd = $param[0];
2536         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2537
2538         for (my $i = 0; $i < scalar (@param); $i++) {
2539             my $p = $param[$i];
2540             next if !$p;
2541             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2542                 my $p = $param[$i+1];
2543                 return 1 if $p && ($p eq $pidfile);
2544                 return undef;
2545             }
2546         }
2547     }
2548     return undef;
2549 }
2550
2551 sub check_running {
2552     my ($vmid, $nocheck, $node) = @_;
2553
2554     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2555
2556     die "unable to find configuration file for VM $vmid - no such machine\n"
2557         if !$nocheck && ! -f $filename;
2558
2559     my $pidfile = pidfile_name($vmid);
2560
2561     if (my $fd = IO::File->new("<$pidfile")) {
2562         my $st = stat($fd);
2563         my $line = <$fd>;
2564         close($fd);
2565
2566         my $mtime = $st->mtime;
2567         if ($mtime > time()) {
2568             warn "file '$filename' modified in future\n";
2569         }
2570
2571         if ($line =~ m/^(\d+)$/) {
2572             my $pid = $1;
2573             if (check_cmdline($pidfile, $pid)) {
2574                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2575                     return $pid;
2576                 }
2577             }
2578         }
2579     }
2580
2581     return undef;
2582 }
2583
2584 sub vzlist {
2585
2586     my $vzlist = config_list();
2587
2588     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2589
2590     while (defined(my $de = $fd->read)) {
2591         next if $de !~ m/^(\d+)\.pid$/;
2592         my $vmid = $1;
2593         next if !defined($vzlist->{$vmid});
2594         if (my $pid = check_running($vmid)) {
2595             $vzlist->{$vmid}->{pid} = $pid;
2596         }
2597     }
2598
2599     return $vzlist;
2600 }
2601
2602 sub disksize {
2603     my ($storecfg, $conf) = @_;
2604
2605     my $bootdisk = $conf->{bootdisk};
2606     return undef if !$bootdisk;
2607     return undef if !is_valid_drivename($bootdisk);
2608
2609     return undef if !$conf->{$bootdisk};
2610
2611     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2612     return undef if !defined($drive);
2613
2614     return undef if drive_is_cdrom($drive);
2615
2616     my $volid = $drive->{file};
2617     return undef if !$volid;
2618
2619     return $drive->{size};
2620 }
2621
2622 my $last_proc_pid_stat;
2623
2624 # get VM status information
2625 # This must be fast and should not block ($full == false)
2626 # We only query KVM using QMP if $full == true (this can be slow)
2627 sub vmstatus {
2628     my ($opt_vmid, $full) = @_;
2629
2630     my $res = {};
2631
2632     my $storecfg = PVE::Storage::config();
2633
2634     my $list = vzlist();
2635     my $defaults = load_defaults();
2636
2637     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2638
2639     my $cpucount = $cpuinfo->{cpus} || 1;
2640
2641     foreach my $vmid (keys %$list) {
2642         next if $opt_vmid && ($vmid ne $opt_vmid);
2643
2644         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2645         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2646
2647         my $d = {};
2648         $d->{pid} = $list->{$vmid}->{pid};
2649
2650         # fixme: better status?
2651         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2652
2653         my $size = disksize($storecfg, $conf);
2654         if (defined($size)) {
2655             $d->{disk} = 0; # no info available
2656             $d->{maxdisk} = $size;
2657         } else {
2658             $d->{disk} = 0;
2659             $d->{maxdisk} = 0;
2660         }
2661
2662         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2663             * ($conf->{cores} || $defaults->{cores});
2664         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2665         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2666
2667         $d->{name} = $conf->{name} || "VM $vmid";
2668         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2669             : $defaults->{memory}*(1024*1024);
2670
2671         if ($conf->{balloon}) {
2672             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2673             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2674                 : $defaults->{shares};
2675         }
2676
2677         $d->{uptime} = 0;
2678         $d->{cpu} = 0;
2679         $d->{mem} = 0;
2680
2681         $d->{netout} = 0;
2682         $d->{netin} = 0;
2683
2684         $d->{diskread} = 0;
2685         $d->{diskwrite} = 0;
2686
2687         $d->{template} = PVE::QemuConfig->is_template($conf);
2688
2689         $d->{serial} = 1 if conf_has_serial($conf);
2690
2691         $res->{$vmid} = $d;
2692     }
2693
2694     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2695     foreach my $dev (keys %$netdev) {
2696         next if $dev !~ m/^tap([1-9]\d*)i/;
2697         my $vmid = $1;
2698         my $d = $res->{$vmid};
2699         next if !$d;
2700
2701         $d->{netout} += $netdev->{$dev}->{receive};
2702         $d->{netin} += $netdev->{$dev}->{transmit};
2703
2704         if ($full) {
2705             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2706             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2707         }
2708
2709     }
2710
2711     my $ctime = gettimeofday;
2712
2713     foreach my $vmid (keys %$list) {
2714
2715         my $d = $res->{$vmid};
2716         my $pid = $d->{pid};
2717         next if !$pid;
2718
2719         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2720         next if !$pstat; # not running
2721
2722         my $used = $pstat->{utime} + $pstat->{stime};
2723
2724         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2725
2726         if ($pstat->{vsize}) {
2727             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2728         }
2729
2730         my $old = $last_proc_pid_stat->{$pid};
2731         if (!$old) {
2732             $last_proc_pid_stat->{$pid} = {
2733                 time => $ctime,
2734                 used => $used,
2735                 cpu => 0,
2736             };
2737             next;
2738         }
2739
2740         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2741
2742         if ($dtime > 1000) {
2743             my $dutime = $used -  $old->{used};
2744
2745             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2746             $last_proc_pid_stat->{$pid} = {
2747                 time => $ctime,
2748                 used => $used,
2749                 cpu => $d->{cpu},
2750             };
2751         } else {
2752             $d->{cpu} = $old->{cpu};
2753         }
2754     }
2755
2756     return $res if !$full;
2757
2758     my $qmpclient = PVE::QMPClient->new();
2759
2760     my $ballooncb = sub {
2761         my ($vmid, $resp) = @_;
2762
2763         my $info = $resp->{'return'};
2764         return if !$info->{max_mem};
2765
2766         my $d = $res->{$vmid};
2767
2768         # use memory assigned to VM
2769         $d->{maxmem} = $info->{max_mem};
2770         $d->{balloon} = $info->{actual};
2771
2772         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2773             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2774             $d->{freemem} = $info->{free_mem};
2775         }
2776
2777         $d->{ballooninfo} = $info;
2778     };
2779
2780     my $blockstatscb = sub {
2781         my ($vmid, $resp) = @_;
2782         my $data = $resp->{'return'} || [];
2783         my $totalrdbytes = 0;
2784         my $totalwrbytes = 0;
2785
2786         for my $blockstat (@$data) {
2787             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2788             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2789
2790             $blockstat->{device} =~ s/drive-//;
2791             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2792         }
2793         $res->{$vmid}->{diskread} = $totalrdbytes;
2794         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2795     };
2796
2797     my $statuscb = sub {
2798         my ($vmid, $resp) = @_;
2799
2800         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2801         # this fails if ballon driver is not loaded, so this must be
2802         # the last commnand (following command are aborted if this fails).
2803         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2804
2805         my $status = 'unknown';
2806         if (!defined($status = $resp->{'return'}->{status})) {
2807             warn "unable to get VM status\n";
2808             return;
2809         }
2810
2811         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2812     };
2813
2814     foreach my $vmid (keys %$list) {
2815         next if $opt_vmid && ($vmid ne $opt_vmid);
2816         next if !$res->{$vmid}->{pid}; # not running
2817         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2818     }
2819
2820     $qmpclient->queue_execute(undef, 2);
2821
2822     foreach my $vmid (keys %$list) {
2823         next if $opt_vmid && ($vmid ne $opt_vmid);
2824         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2825     }
2826
2827     return $res;
2828 }
2829
2830 sub foreach_drive {
2831     my ($conf, $func, @param) = @_;
2832
2833     foreach my $ds (valid_drive_names()) {
2834         next if !defined($conf->{$ds});
2835
2836         my $drive = parse_drive($ds, $conf->{$ds});
2837         next if !$drive;
2838
2839         &$func($ds, $drive, @param);
2840     }
2841 }
2842
2843 sub foreach_volid {
2844     my ($conf, $func, @param) = @_;
2845
2846     my $volhash = {};
2847
2848     my $test_volid = sub {
2849         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
2850
2851         return if !$volid;
2852
2853         $volhash->{$volid}->{cdrom} //= 1;
2854         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2855
2856         $volhash->{$volid}->{replicate} //= 0;
2857         $volhash->{$volid}->{replicate} = 1 if $replicate;
2858
2859         $volhash->{$volid}->{shared} //= 0;
2860         $volhash->{$volid}->{shared} = 1 if $shared;
2861
2862         $volhash->{$volid}->{referenced_in_config} //= 0;
2863         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
2864
2865         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
2866             if defined($snapname);
2867     };
2868
2869     foreach_drive($conf, sub {
2870         my ($ds, $drive) = @_;
2871         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
2872     });
2873
2874     foreach my $snapname (keys %{$conf->{snapshots}}) {
2875         my $snap = $conf->{snapshots}->{$snapname};
2876         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
2877         foreach_drive($snap, sub {
2878             my ($ds, $drive) = @_;
2879             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
2880         });
2881     }
2882
2883     foreach my $volid (keys %$volhash) {
2884         &$func($volid, $volhash->{$volid}, @param);
2885     }
2886 }
2887
2888 sub conf_has_serial {
2889     my ($conf) = @_;
2890
2891     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2892         if ($conf->{"serial$i"}) {
2893             return 1;
2894         }
2895     }
2896
2897     return 0;
2898 }
2899
2900 sub vga_conf_has_spice {
2901     my ($vga) = @_;
2902
2903     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2904
2905     return $1 || 1;
2906 }
2907
2908 sub config_to_command {
2909     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2910
2911     my $cmd = [];
2912     my $globalFlags = [];
2913     my $machineFlags = [];
2914     my $rtcFlags = [];
2915     my $cpuFlags = [];
2916     my $devices = [];
2917     my $pciaddr = '';
2918     my $bridges = {};
2919     my $kvmver = kvm_user_version();
2920     my $vernum = 0; # unknown
2921     my $ostype = $conf->{ostype};
2922     my $winversion = windows_version($ostype);
2923     my $kvm = $conf->{kvm} // 1;
2924
2925     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
2926
2927     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2928         $vernum = $1*1000000+$2*1000;
2929     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2930         $vernum = $1*1000000+$2*1000+$3;
2931     }
2932
2933     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2934
2935     my $have_ovz = -f '/proc/vz/vestat';
2936
2937     my $q35 = machine_type_is_q35($conf);
2938     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2939     my $machine_type = $forcemachine || $conf->{machine};
2940     my $use_old_bios_files = undef;
2941     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2942
2943     my $cpuunits = defined($conf->{cpuunits}) ?
2944             $conf->{cpuunits} : $defaults->{cpuunits};
2945
2946     push @$cmd, '/usr/bin/kvm';
2947
2948     push @$cmd, '-id', $vmid;
2949
2950     my $use_virtio = 0;
2951
2952     my $qmpsocket = qmp_socket($vmid);
2953     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2954     push @$cmd, '-mon', "chardev=qmp,mode=control";
2955
2956
2957     push @$cmd, '-pidfile' , pidfile_name($vmid);
2958
2959     push @$cmd, '-daemonize';
2960
2961     if ($conf->{smbios1}) {
2962         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2963     }
2964
2965     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2966         die "uefi base image not found\n" if ! -f $OVMF_CODE;
2967
2968         my $path;
2969         my $format;
2970         if (my $efidisk = $conf->{efidisk0}) {
2971             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
2972             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2973             $format = $d->{format};
2974             if ($storeid) {
2975                 $path = PVE::Storage::path($storecfg, $d->{file});
2976                 if (!defined($format)) {
2977                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2978                     $format = qemu_img_format($scfg, $volname);
2979                 }
2980             } else {
2981                 $path = $d->{file};
2982                 die "efidisk format must be specified\n"
2983                     if !defined($format);
2984             }
2985         } else {
2986             warn "no efidisk configured! Using temporary efivars disk.\n";
2987             $path = "/tmp/$vmid-ovmf.fd";
2988             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
2989             $format = 'raw';
2990         }
2991
2992         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
2993         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
2994     }
2995
2996
2997     # add usb controllers
2998     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2999     push @$devices, @usbcontrollers if @usbcontrollers;
3000     my $vga = $conf->{vga};
3001
3002     my $qxlnum = vga_conf_has_spice($vga);
3003     $vga = 'qxl' if $qxlnum;
3004
3005     if (!$vga) {
3006         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3007             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3008         } else {
3009             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
3010         }
3011     }
3012
3013     # enable absolute mouse coordinates (needed by vnc)
3014     my $tablet;
3015     if (defined($conf->{tablet})) {
3016         $tablet = $conf->{tablet};
3017     } else {
3018         $tablet = $defaults->{tablet};
3019         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3020         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3021     }
3022
3023     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3024
3025     my $kvm_off = 0;
3026     my $gpu_passthrough;
3027
3028     # host pci devices
3029     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3030         my $d = parse_hostpci($conf->{"hostpci$i"});
3031         next if !$d;
3032
3033         my $pcie = $d->{pcie};
3034         if($pcie){
3035             die "q35 machine model is not enabled" if !$q35;
3036             $pciaddr = print_pcie_addr("hostpci$i");
3037         }else{
3038             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3039         }
3040
3041         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3042         my $romfile = $d->{romfile};
3043
3044         my $xvga = '';
3045         if ($d->{'x-vga'}) {
3046             $xvga = ',x-vga=on';
3047             $kvm_off = 1;
3048             $vga = 'none';
3049             $gpu_passthrough = 1;
3050
3051             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3052                 $xvga = "";
3053             }
3054         }
3055         my $pcidevices = $d->{pciid};
3056         my $multifunction = 1 if @$pcidevices > 1;
3057
3058         my $j=0;
3059         foreach my $pcidevice (@$pcidevices) {
3060
3061             my $id = "hostpci$i";
3062             $id .= ".$j" if $multifunction;
3063             my $addr = $pciaddr;
3064             $addr .= ".$j" if $multifunction;
3065             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3066
3067             if($j == 0){
3068                 $devicestr .= "$rombar$xvga";
3069                 $devicestr .= ",multifunction=on" if $multifunction;
3070                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3071             }
3072
3073             push @$devices, '-device', $devicestr;
3074             $j++;
3075         }
3076     }
3077
3078     # usb devices
3079     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3080     push @$devices, @usbdevices if @usbdevices;
3081     # serial devices
3082     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3083         if (my $path = $conf->{"serial$i"}) {
3084             if ($path eq 'socket') {
3085                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3086                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3087                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3088             } else {
3089                 die "no such serial device\n" if ! -c $path;
3090                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3091                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3092             }
3093         }
3094     }
3095
3096     # parallel devices
3097     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3098         if (my $path = $conf->{"parallel$i"}) {
3099             die "no such parallel device\n" if ! -c $path;
3100             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3101             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3102             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3103         }
3104     }
3105
3106     my $vmname = $conf->{name} || "vm$vmid";
3107
3108     push @$cmd, '-name', $vmname;
3109
3110     my $sockets = 1;
3111     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3112     $sockets = $conf->{sockets} if  $conf->{sockets};
3113
3114     my $cores = $conf->{cores} || 1;
3115
3116     my $maxcpus = $sockets * $cores;
3117
3118     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3119
3120     my $allowed_vcpus = $cpuinfo->{cpus};
3121
3122     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3123         if ($allowed_vcpus < $maxcpus);
3124
3125     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3126
3127         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3128         for (my $i = 2; $i <= $vcpus; $i++)  {
3129             my $cpustr = print_cpu_device($conf,$i);
3130             push @$cmd, '-device', $cpustr;
3131         }
3132
3133     } else {
3134
3135         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3136     }
3137     push @$cmd, '-nodefaults';
3138
3139     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3140
3141     my $bootindex_hash = {};
3142     my $i = 1;
3143     foreach my $o (split(//, $bootorder)) {
3144         $bootindex_hash->{$o} = $i*100;
3145         $i++;
3146     }
3147
3148     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3149
3150     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3151
3152     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3153
3154     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3155
3156     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3157         my $socket = vnc_socket($vmid);
3158         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3159     } else {
3160         push @$cmd, '-nographic';
3161     }
3162
3163     # time drift fix
3164     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3165
3166     my $useLocaltime = $conf->{localtime};
3167
3168     if ($winversion >= 5) { # windows
3169         $useLocaltime = 1 if !defined($conf->{localtime});
3170
3171         # use time drift fix when acpi is enabled
3172         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3173             $tdf = 1 if !defined($conf->{tdf});
3174         }
3175     }
3176
3177     if ($winversion >= 6) {
3178         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3179         push @$cmd, '-no-hpet';
3180     }
3181
3182     push @$rtcFlags, 'driftfix=slew' if $tdf;
3183
3184     if (!$kvm) {
3185         push @$machineFlags, 'accel=tcg';
3186     }
3187
3188     if ($machine_type) {
3189         push @$machineFlags, "type=${machine_type}";
3190     }
3191
3192     if ($conf->{startdate}) {
3193         push @$rtcFlags, "base=$conf->{startdate}";
3194     } elsif ($useLocaltime) {
3195         push @$rtcFlags, 'base=localtime';
3196     }
3197
3198     my $cpu = $kvm ? "kvm64" : "qemu64";
3199     if (my $cputype = $conf->{cpu}) {
3200         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3201             or die "Cannot parse cpu description: $cputype\n";
3202         $cpu = $cpuconf->{cputype};
3203         $kvm_off = 1 if $cpuconf->{hidden};
3204
3205         if (defined(my $flags = $cpuconf->{flags})) {
3206             push @$cpuFlags, split(";", $flags);
3207         }
3208     }
3209
3210     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3211
3212     push @$cpuFlags , '-x2apic'
3213         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3214
3215     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3216
3217     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3218
3219     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3220
3221         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3222         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3223     }
3224
3225     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3226
3227     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3228
3229     push @$cpuFlags, 'kvm=off' if $kvm_off;
3230
3231     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3232         die "internal error"; # should not happen
3233
3234     push @$cpuFlags, "vendor=${cpu_vendor}"
3235         if $cpu_vendor ne 'default';
3236
3237     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3238
3239     push @$cmd, '-cpu', $cpu;
3240
3241     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3242
3243     push @$cmd, '-S' if $conf->{freeze};
3244
3245     # set keyboard layout
3246     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3247     push @$cmd, '-k', $kb if $kb;
3248
3249     # enable sound
3250     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3251     #push @$cmd, '-soundhw', 'es1370';
3252     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3253
3254     if($conf->{agent}) {
3255         my $qgasocket = qmp_socket($vmid, 1);
3256         my $pciaddr = print_pci_addr("qga0", $bridges);
3257         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3258         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3259         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3260     }
3261
3262     my $spice_port;
3263
3264     if ($qxlnum) {
3265         if ($qxlnum > 1) {
3266             if ($winversion){
3267                 for(my $i = 1; $i < $qxlnum; $i++){
3268                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3269                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3270                 }
3271             } else {
3272                 # assume other OS works like Linux
3273                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3274                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3275             }
3276         }
3277
3278         my $pciaddr = print_pci_addr("spice", $bridges);
3279
3280         my $nodename = PVE::INotify::nodename();
3281         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3282         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3283         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3284         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3285         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3286
3287         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3288
3289         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3290         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3291         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3292     }
3293
3294     # enable balloon by default, unless explicitly disabled
3295     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3296         $pciaddr = print_pci_addr("balloon0", $bridges);
3297         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3298     }
3299
3300     if ($conf->{watchdog}) {
3301         my $wdopts = parse_watchdog($conf->{watchdog});
3302         $pciaddr = print_pci_addr("watchdog", $bridges);
3303         my $watchdog = $wdopts->{model} || 'i6300esb';
3304         push @$devices, '-device', "$watchdog$pciaddr";
3305         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3306     }
3307
3308     my $vollist = [];
3309     my $scsicontroller = {};
3310     my $ahcicontroller = {};
3311     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3312
3313     # Add iscsi initiator name if available
3314     if (my $initiator = get_initiator_name()) {
3315         push @$devices, '-iscsi', "initiator-name=$initiator";
3316     }
3317
3318     foreach_drive($conf, sub {
3319         my ($ds, $drive) = @_;
3320
3321         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3322             push @$vollist, $drive->{file};
3323         }
3324
3325         # ignore efidisk here, already added in bios/fw handling code above
3326         return if $drive->{interface} eq 'efidisk';
3327
3328         $use_virtio = 1 if $ds =~ m/^virtio/;
3329
3330         if (drive_is_cdrom ($drive)) {
3331             if ($bootindex_hash->{d}) {
3332                 $drive->{bootindex} = $bootindex_hash->{d};
3333                 $bootindex_hash->{d} += 1;
3334             }
3335         } else {
3336             if ($bootindex_hash->{c}) {
3337                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3338                 $bootindex_hash->{c} += 1;
3339             }
3340         }
3341
3342         if($drive->{interface} eq 'virtio'){
3343            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3344         }
3345
3346         if ($drive->{interface} eq 'scsi') {
3347
3348             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3349
3350             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3351             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3352
3353             my $iothread = '';
3354             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3355                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3356                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3357             } elsif ($drive->{iothread}) {
3358                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3359             }
3360
3361             my $queues = '';
3362             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3363                 $queues = ",num_queues=$drive->{queues}";
3364             }
3365
3366             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3367             $scsicontroller->{$controller}=1;
3368         }
3369
3370         if ($drive->{interface} eq 'sata') {
3371            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3372            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3373            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3374            $ahcicontroller->{$controller}=1;
3375         }
3376
3377         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3378         push @$devices, '-drive',$drive_cmd;
3379         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3380     });
3381
3382     for (my $i = 0; $i < $MAX_NETS; $i++) {
3383          next if !$conf->{"net$i"};
3384          my $d = parse_net($conf->{"net$i"});
3385          next if !$d;
3386
3387          $use_virtio = 1 if $d->{model} eq 'virtio';
3388
3389          if ($bootindex_hash->{n}) {
3390             $d->{bootindex} = $bootindex_hash->{n};
3391             $bootindex_hash->{n} += 1;
3392          }
3393
3394          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3395          push @$devices, '-netdev', $netdevfull;
3396
3397          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3398          push @$devices, '-device', $netdevicefull;
3399     }
3400
3401     if (!$q35) {
3402         # add pci bridges
3403         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3404            $bridges->{1} = 1;
3405            $bridges->{2} = 1;
3406         }
3407
3408         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3409
3410         while (my ($k, $v) = each %$bridges) {
3411             $pciaddr = print_pci_addr("pci.$k");
3412             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3413         }
3414     }
3415
3416     # add custom args
3417     if ($conf->{args}) {
3418         my $aa = PVE::Tools::split_args($conf->{args});
3419         push @$cmd, @$aa;
3420     }
3421
3422     push @$cmd, @$devices;
3423     push @$cmd, '-rtc', join(',', @$rtcFlags)
3424         if scalar(@$rtcFlags);
3425     push @$cmd, '-machine', join(',', @$machineFlags)
3426         if scalar(@$machineFlags);
3427     push @$cmd, '-global', join(',', @$globalFlags)
3428         if scalar(@$globalFlags);
3429
3430     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3431 }
3432
3433 sub vnc_socket {
3434     my ($vmid) = @_;
3435     return "${var_run_tmpdir}/$vmid.vnc";
3436 }
3437
3438 sub spice_port {
3439     my ($vmid) = @_;
3440
3441     my $res = vm_mon_cmd($vmid, 'query-spice');
3442
3443     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3444 }
3445
3446 sub qmp_socket {
3447     my ($vmid, $qga) = @_;
3448     my $sockettype = $qga ? 'qga' : 'qmp';
3449     return "${var_run_tmpdir}/$vmid.$sockettype";
3450 }
3451
3452 sub pidfile_name {
3453     my ($vmid) = @_;
3454     return "${var_run_tmpdir}/$vmid.pid";
3455 }
3456
3457 sub vm_devices_list {
3458     my ($vmid) = @_;
3459
3460     my $res = vm_mon_cmd($vmid, 'query-pci');
3461     my $devices = {};
3462     foreach my $pcibus (@$res) {
3463         foreach my $device (@{$pcibus->{devices}}) {
3464             next if !$device->{'qdev_id'};
3465             if ($device->{'pci_bridge'}) {
3466                 $devices->{$device->{'qdev_id'}} = 1;
3467                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3468                     next if !$bridge_device->{'qdev_id'};
3469                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3470                     $devices->{$device->{'qdev_id'}}++;
3471                 }
3472             } else {
3473                 $devices->{$device->{'qdev_id'}} = 1;
3474             }
3475         }
3476     }
3477
3478     my $resblock = vm_mon_cmd($vmid, 'query-block');
3479     foreach my $block (@$resblock) {
3480         if($block->{device} =~ m/^drive-(\S+)/){
3481                 $devices->{$1} = 1;
3482         }
3483     }
3484
3485     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3486     foreach my $mice (@$resmice) {
3487         if ($mice->{name} eq 'QEMU HID Tablet') {
3488             $devices->{tablet} = 1;
3489             last;
3490         }
3491     }
3492
3493     # for usb devices there is no query-usb
3494     # but we can iterate over the entries in
3495     # qom-list path=/machine/peripheral
3496     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3497     foreach my $per (@$resperipheral) {
3498         if ($per->{name} =~ m/^usb\d+$/) {
3499             $devices->{$per->{name}} = 1;
3500         }
3501     }
3502
3503     return $devices;
3504 }
3505
3506 sub vm_deviceplug {
3507     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3508
3509     my $q35 = machine_type_is_q35($conf);
3510
3511     my $devices_list = vm_devices_list($vmid);
3512     return 1 if defined($devices_list->{$deviceid});
3513
3514     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3515
3516     if ($deviceid eq 'tablet') {
3517
3518         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3519
3520     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3521
3522         die "usb hotplug currently not reliable\n";
3523         # since we can't reliably hot unplug all added usb devices
3524         # and usb passthrough disables live migration
3525         # we disable usb hotplugging for now
3526         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3527
3528     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3529
3530         qemu_iothread_add($vmid, $deviceid, $device);
3531
3532         qemu_driveadd($storecfg, $vmid, $device);
3533         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3534
3535         qemu_deviceadd($vmid, $devicefull);
3536         eval { qemu_deviceaddverify($vmid, $deviceid); };
3537         if (my $err = $@) {
3538             eval { qemu_drivedel($vmid, $deviceid); };
3539             warn $@ if $@;
3540             die $err;
3541         }
3542
3543     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3544
3545
3546         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3547         my $pciaddr = print_pci_addr($deviceid);
3548         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3549
3550         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3551
3552         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3553             qemu_iothread_add($vmid, $deviceid, $device);
3554             $devicefull .= ",iothread=iothread-$deviceid";
3555         }
3556
3557         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3558             $devicefull .= ",num_queues=$device->{queues}";
3559         }
3560
3561         qemu_deviceadd($vmid, $devicefull);
3562         qemu_deviceaddverify($vmid, $deviceid);
3563
3564     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3565
3566         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3567         qemu_driveadd($storecfg, $vmid, $device);
3568
3569         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3570         eval { qemu_deviceadd($vmid, $devicefull); };
3571         if (my $err = $@) {
3572             eval { qemu_drivedel($vmid, $deviceid); };
3573             warn $@ if $@;
3574             die $err;
3575         }
3576
3577     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3578
3579         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3580
3581         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3582         my $use_old_bios_files = undef;
3583         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3584
3585         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3586         qemu_deviceadd($vmid, $netdevicefull);
3587         eval { qemu_deviceaddverify($vmid, $deviceid); };
3588         if (my $err = $@) {
3589             eval { qemu_netdevdel($vmid, $deviceid); };
3590             warn $@ if $@;
3591             die $err;
3592         }
3593
3594     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3595
3596         my $bridgeid = $2;
3597         my $pciaddr = print_pci_addr($deviceid);
3598         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3599
3600         qemu_deviceadd($vmid, $devicefull);
3601         qemu_deviceaddverify($vmid, $deviceid);
3602
3603     } else {
3604         die "can't hotplug device '$deviceid'\n";
3605     }
3606
3607     return 1;
3608 }
3609
3610 # fixme: this should raise exceptions on error!
3611 sub vm_deviceunplug {
3612     my ($vmid, $conf, $deviceid) = @_;
3613
3614     my $devices_list = vm_devices_list($vmid);
3615     return 1 if !defined($devices_list->{$deviceid});
3616
3617     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3618
3619     if ($deviceid eq 'tablet') {
3620
3621         qemu_devicedel($vmid, $deviceid);
3622
3623     } elsif ($deviceid =~ m/^usb\d+$/) {
3624
3625         die "usb hotplug currently not reliable\n";
3626         # when unplugging usb devices this way,
3627         # there may be remaining usb controllers/hubs
3628         # so we disable it for now
3629         qemu_devicedel($vmid, $deviceid);
3630         qemu_devicedelverify($vmid, $deviceid);
3631
3632     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3633
3634         qemu_devicedel($vmid, $deviceid);
3635         qemu_devicedelverify($vmid, $deviceid);
3636         qemu_drivedel($vmid, $deviceid);
3637         qemu_iothread_del($conf, $vmid, $deviceid);
3638
3639     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3640
3641         qemu_devicedel($vmid, $deviceid);
3642         qemu_devicedelverify($vmid, $deviceid);
3643         qemu_iothread_del($conf, $vmid, $deviceid);
3644
3645     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3646
3647         #qemu 2.3 segfault on drive_del with virtioscsi + iothread
3648         my $device = parse_drive($deviceid, $conf->{$deviceid});
3649         die "virtioscsi with iothread is not hot-unplugglable currently" if $device->{iothread};
3650
3651         qemu_devicedel($vmid, $deviceid);
3652         qemu_drivedel($vmid, $deviceid);
3653         qemu_deletescsihw($conf, $vmid, $deviceid);
3654
3655     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3656
3657         qemu_devicedel($vmid, $deviceid);
3658         qemu_devicedelverify($vmid, $deviceid);
3659         qemu_netdevdel($vmid, $deviceid);
3660
3661     } else {
3662         die "can't unplug device '$deviceid'\n";
3663     }
3664
3665     return 1;
3666 }
3667
3668 sub qemu_deviceadd {
3669     my ($vmid, $devicefull) = @_;
3670
3671     $devicefull = "driver=".$devicefull;
3672     my %options =  split(/[=,]/, $devicefull);
3673
3674     vm_mon_cmd($vmid, "device_add" , %options);
3675 }
3676
3677 sub qemu_devicedel {
3678     my ($vmid, $deviceid) = @_;
3679
3680     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3681 }
3682
3683 sub qemu_iothread_add {
3684     my($vmid, $deviceid, $device) = @_;
3685
3686     if ($device->{iothread}) {
3687         my $iothreads = vm_iothreads_list($vmid);
3688         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3689     }
3690 }
3691
3692 sub qemu_iothread_del {
3693     my($conf, $vmid, $deviceid) = @_;
3694
3695     my $device = parse_drive($deviceid, $conf->{$deviceid});
3696     if ($device->{iothread}) {
3697         my $iothreads = vm_iothreads_list($vmid);
3698         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3699     }
3700 }
3701
3702 sub qemu_objectadd {
3703     my($vmid, $objectid, $qomtype) = @_;
3704
3705     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3706
3707     return 1;
3708 }
3709
3710 sub qemu_objectdel {
3711     my($vmid, $objectid) = @_;
3712
3713     vm_mon_cmd($vmid, "object-del", id => $objectid);
3714
3715     return 1;
3716 }
3717
3718 sub qemu_driveadd {
3719     my ($storecfg, $vmid, $device) = @_;
3720
3721     my $drive = print_drive_full($storecfg, $vmid, $device);
3722     $drive =~ s/\\/\\\\/g;
3723     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3724
3725     # If the command succeeds qemu prints: "OK"
3726     return 1 if $ret =~ m/OK/s;
3727
3728     die "adding drive failed: $ret\n";
3729 }
3730
3731 sub qemu_drivedel {
3732     my($vmid, $deviceid) = @_;
3733
3734     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3735     $ret =~ s/^\s+//;
3736
3737     return 1 if $ret eq "";
3738
3739     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3740     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3741
3742     die "deleting drive $deviceid failed : $ret\n";
3743 }
3744
3745 sub qemu_deviceaddverify {
3746     my ($vmid, $deviceid) = @_;
3747
3748     for (my $i = 0; $i <= 5; $i++) {
3749          my $devices_list = vm_devices_list($vmid);
3750          return 1 if defined($devices_list->{$deviceid});
3751          sleep 1;
3752     }
3753
3754     die "error on hotplug device '$deviceid'\n";
3755 }
3756
3757
3758 sub qemu_devicedelverify {
3759     my ($vmid, $deviceid) = @_;
3760
3761     # need to verify that the device is correctly removed as device_del
3762     # is async and empty return is not reliable
3763
3764     for (my $i = 0; $i <= 5; $i++) {
3765          my $devices_list = vm_devices_list($vmid);
3766          return 1 if !defined($devices_list->{$deviceid});
3767          sleep 1;
3768     }
3769
3770     die "error on hot-unplugging device '$deviceid'\n";
3771 }
3772
3773 sub qemu_findorcreatescsihw {
3774     my ($storecfg, $conf, $vmid, $device) = @_;
3775
3776     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3777
3778     my $scsihwid="$controller_prefix$controller";
3779     my $devices_list = vm_devices_list($vmid);
3780
3781     if(!defined($devices_list->{$scsihwid})) {
3782         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3783     }
3784
3785     return 1;
3786 }
3787
3788 sub qemu_deletescsihw {
3789     my ($conf, $vmid, $opt) = @_;
3790
3791     my $device = parse_drive($opt, $conf->{$opt});
3792
3793     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3794         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3795         return 1;
3796     }
3797
3798     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3799
3800     my $devices_list = vm_devices_list($vmid);
3801     foreach my $opt (keys %{$devices_list}) {
3802         if (PVE::QemuServer::is_valid_drivename($opt)) {
3803             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3804             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3805                 return 1;
3806             }
3807         }
3808     }
3809
3810     my $scsihwid="scsihw$controller";
3811
3812     vm_deviceunplug($vmid, $conf, $scsihwid);
3813
3814     return 1;
3815 }
3816
3817 sub qemu_add_pci_bridge {
3818     my ($storecfg, $conf, $vmid, $device) = @_;
3819
3820     my $bridges = {};
3821
3822     my $bridgeid;
3823
3824     print_pci_addr($device, $bridges);
3825
3826     while (my ($k, $v) = each %$bridges) {
3827         $bridgeid = $k;
3828     }
3829     return 1 if !defined($bridgeid) || $bridgeid < 1;
3830
3831     my $bridge = "pci.$bridgeid";
3832     my $devices_list = vm_devices_list($vmid);
3833
3834     if (!defined($devices_list->{$bridge})) {
3835         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3836     }
3837
3838     return 1;
3839 }
3840
3841 sub qemu_set_link_status {
3842     my ($vmid, $device, $up) = @_;
3843
3844     vm_mon_cmd($vmid, "set_link", name => $device,
3845                up => $up ? JSON::true : JSON::false);
3846 }
3847
3848 sub qemu_netdevadd {
3849     my ($vmid, $conf, $device, $deviceid) = @_;
3850
3851     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3852     my %options =  split(/[=,]/, $netdev);
3853
3854     vm_mon_cmd($vmid, "netdev_add",  %options);
3855     return 1;
3856 }
3857
3858 sub qemu_netdevdel {
3859     my ($vmid, $deviceid) = @_;
3860
3861     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3862 }
3863
3864 sub qemu_usb_hotplug {
3865     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3866
3867     return if !$device;
3868
3869     # remove the old one first
3870     vm_deviceunplug($vmid, $conf, $deviceid);
3871
3872     # check if xhci controller is necessary and available
3873     if ($device->{usb3}) {
3874
3875         my $devicelist = vm_devices_list($vmid);
3876
3877         if (!$devicelist->{xhci}) {
3878             my $pciaddr = print_pci_addr("xhci");
3879             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3880         }
3881     }
3882     my $d = parse_usb_device($device->{host});
3883     $d->{usb3} = $device->{usb3};
3884
3885     # add the new one
3886     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3887 }
3888
3889 sub qemu_cpu_hotplug {
3890     my ($vmid, $conf, $vcpus) = @_;
3891
3892     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3893
3894     my $sockets = 1;
3895     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3896     $sockets = $conf->{sockets} if  $conf->{sockets};
3897     my $cores = $conf->{cores} || 1;
3898     my $maxcpus = $sockets * $cores;
3899
3900     $vcpus = $maxcpus if !$vcpus;
3901
3902     die "you can't add more vcpus than maxcpus\n"
3903         if $vcpus > $maxcpus;
3904
3905     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3906
3907     if ($vcpus < $currentvcpus) {
3908
3909         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3910
3911             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3912                 qemu_devicedel($vmid, "cpu$i");
3913                 my $retry = 0;
3914                 my $currentrunningvcpus = undef;
3915                 while (1) {
3916                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3917                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3918                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3919                     $retry++;
3920                     sleep 1;
3921                 }
3922                 #update conf after each succesfull cpu unplug
3923                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3924                 PVE::QemuConfig->write_config($vmid, $conf);
3925             }
3926         } else {
3927             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3928         }
3929
3930         return;
3931     }
3932
3933     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3934     die "vcpus in running vm does not match its configuration\n"
3935         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3936
3937     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3938
3939         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3940             my $cpustr = print_cpu_device($conf, $i);
3941             qemu_deviceadd($vmid, $cpustr);
3942
3943             my $retry = 0;
3944             my $currentrunningvcpus = undef;
3945             while (1) {
3946                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3947                 last if scalar(@{$currentrunningvcpus}) == $i;
3948                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3949                 sleep 1;
3950                 $retry++;
3951             }
3952             #update conf after each succesfull cpu hotplug
3953             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3954             PVE::QemuConfig->write_config($vmid, $conf);
3955         }
3956     } else {
3957
3958         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3959             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3960         }
3961     }
3962 }
3963
3964 sub qemu_block_set_io_throttle {
3965     my ($vmid, $deviceid,
3966         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3967         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3968         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3969         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3970
3971     return if !check_running($vmid) ;
3972
3973     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3974         bps => int($bps),
3975         bps_rd => int($bps_rd),
3976         bps_wr => int($bps_wr),
3977         iops => int($iops),
3978         iops_rd => int($iops_rd),
3979         iops_wr => int($iops_wr),
3980         bps_max => int($bps_max),
3981         bps_rd_max => int($bps_rd_max),
3982         bps_wr_max => int($bps_wr_max),
3983         iops_max => int($iops_max),
3984         iops_rd_max => int($iops_rd_max),
3985         iops_wr_max => int($iops_wr_max),
3986         bps_max_length => int($bps_max_length),
3987         bps_rd_max_length => int($bps_rd_max_length),
3988         bps_wr_max_length => int($bps_wr_max_length),
3989         iops_max_length => int($iops_max_length),
3990         iops_rd_max_length => int($iops_rd_max_length),
3991         iops_wr_max_length => int($iops_wr_max_length),
3992     );
3993
3994 }
3995
3996 # old code, only used to shutdown old VM after update
3997 sub __read_avail {
3998     my ($fh, $timeout) = @_;
3999
4000     my $sel = new IO::Select;
4001     $sel->add($fh);
4002
4003     my $res = '';
4004     my $buf;
4005
4006     my @ready;
4007     while (scalar (@ready = $sel->can_read($timeout))) {
4008         my $count;
4009         if ($count = $fh->sysread($buf, 8192)) {
4010             if ($buf =~ /^(.*)\(qemu\) $/s) {
4011                 $res .= $1;
4012                 last;
4013             } else {
4014                 $res .= $buf;
4015             }
4016         } else {
4017             if (!defined($count)) {
4018                 die "$!\n";
4019             }
4020             last;
4021         }
4022     }
4023
4024     die "monitor read timeout\n" if !scalar(@ready);
4025
4026     return $res;
4027 }
4028
4029 # old code, only used to shutdown old VM after update
4030 sub vm_monitor_command {
4031     my ($vmid, $cmdstr, $nocheck) = @_;
4032
4033     my $res;
4034
4035     eval {
4036         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4037
4038         my $sname = "${var_run_tmpdir}/$vmid.mon";
4039
4040         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
4041             die "unable to connect to VM $vmid socket - $!\n";
4042
4043         my $timeout = 3;
4044
4045         # hack: migrate sometime blocks the monitor (when migrate_downtime
4046         # is set)
4047         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4048             $timeout = 60*60; # 1 hour
4049         }
4050
4051         # read banner;
4052         my $data = __read_avail($sock, $timeout);
4053
4054         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
4055             die "got unexpected qemu monitor banner\n";
4056         }
4057
4058         my $sel = new IO::Select;
4059         $sel->add($sock);
4060
4061         if (!scalar(my @ready = $sel->can_write($timeout))) {
4062             die "monitor write error - timeout";
4063         }
4064
4065         my $fullcmd = "$cmdstr\r";
4066
4067         # syslog('info', "VM $vmid monitor command: $cmdstr");
4068
4069         my $b;
4070         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
4071             die "monitor write error - $!";
4072         }
4073
4074         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4075
4076         $timeout = 20;
4077
4078         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4079             $timeout = 60*60; # 1 hour
4080         } elsif ($cmdstr =~ m/^(eject|change)/) {
4081             $timeout = 60; # note: cdrom mount command is slow
4082         }
4083         if ($res = __read_avail($sock, $timeout)) {
4084
4085             my @lines = split("\r?\n", $res);
4086
4087             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4088
4089             $res = join("\n", @lines);
4090             $res .= "\n";
4091         }
4092     };
4093
4094     my $err = $@;
4095
4096     if ($err) {
4097         syslog("err", "VM $vmid monitor command failed - $err");
4098         die $err;
4099     }
4100
4101     return $res;
4102 }
4103
4104 sub qemu_block_resize {
4105     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4106
4107     my $running = check_running($vmid);
4108
4109     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4110
4111     return if !$running;
4112
4113     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4114
4115 }
4116
4117 sub qemu_volume_snapshot {
4118     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4119
4120     my $running = check_running($vmid);
4121
4122     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4123         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4124     } else {
4125         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4126     }
4127 }
4128
4129 sub qemu_volume_snapshot_delete {
4130     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4131
4132     my $running = check_running($vmid);
4133
4134     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4135         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4136     } else {
4137         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4138     }
4139 }
4140
4141 sub set_migration_caps {
4142     my ($vmid) = @_;
4143
4144     my $cap_ref = [];
4145
4146     my $enabled_cap = {
4147         "auto-converge" => 1,
4148         "xbzrle" => 1,
4149         "x-rdma-pin-all" => 0,
4150         "zero-blocks" => 0,
4151         "compress" => 0
4152     };
4153
4154     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4155
4156     for my $supported_capability (@$supported_capabilities) {
4157         push @$cap_ref, {
4158             capability => $supported_capability->{capability},
4159             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4160         };
4161     }
4162
4163     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4164 }
4165
4166 my $fast_plug_option = {
4167     'lock' => 1,
4168     'name' => 1,
4169     'onboot' => 1,
4170     'shares' => 1,
4171     'startup' => 1,
4172     'description' => 1,
4173     'protection' => 1,
4174     'vmstatestorage' => 1,
4175 };
4176
4177 # hotplug changes in [PENDING]
4178 # $selection hash can be used to only apply specified options, for
4179 # example: { cores => 1 } (only apply changed 'cores')
4180 # $errors ref is used to return error messages
4181 sub vmconfig_hotplug_pending {
4182     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4183
4184     my $defaults = load_defaults();
4185
4186     # commit values which do not have any impact on running VM first
4187     # Note: those option cannot raise errors, we we do not care about
4188     # $selection and always apply them.
4189
4190     my $add_error = sub {
4191         my ($opt, $msg) = @_;
4192         $errors->{$opt} = "hotplug problem - $msg";
4193     };
4194
4195     my $changes = 0;
4196     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4197         if ($fast_plug_option->{$opt}) {
4198             $conf->{$opt} = $conf->{pending}->{$opt};
4199             delete $conf->{pending}->{$opt};
4200             $changes = 1;
4201         }
4202     }
4203
4204     if ($changes) {
4205         PVE::QemuConfig->write_config($vmid, $conf);
4206         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4207     }
4208
4209     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4210
4211     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4212     while (my ($opt, $force) = each %$pending_delete_hash) {
4213         next if $selection && !$selection->{$opt};
4214         eval {
4215             if ($opt eq 'hotplug') {
4216                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4217             } elsif ($opt eq 'tablet') {
4218                 die "skip\n" if !$hotplug_features->{usb};
4219                 if ($defaults->{tablet}) {
4220                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4221                 } else {
4222                     vm_deviceunplug($vmid, $conf, $opt);
4223                 }
4224             } elsif ($opt =~ m/^usb\d+/) {
4225                 die "skip\n";
4226                 # since we cannot reliably hot unplug usb devices
4227                 # we are disabling it
4228                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4229                 vm_deviceunplug($vmid, $conf, $opt);
4230             } elsif ($opt eq 'vcpus') {
4231                 die "skip\n" if !$hotplug_features->{cpu};
4232                 qemu_cpu_hotplug($vmid, $conf, undef);
4233             } elsif ($opt eq 'balloon') {
4234                 # enable balloon device is not hotpluggable
4235                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4236             } elsif ($fast_plug_option->{$opt}) {
4237                 # do nothing
4238             } elsif ($opt =~ m/^net(\d+)$/) {
4239                 die "skip\n" if !$hotplug_features->{network};
4240                 vm_deviceunplug($vmid, $conf, $opt);
4241             } elsif (is_valid_drivename($opt)) {
4242                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4243                 vm_deviceunplug($vmid, $conf, $opt);
4244                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4245             } elsif ($opt =~ m/^memory$/) {
4246                 die "skip\n" if !$hotplug_features->{memory};
4247                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4248             } elsif ($opt eq 'cpuunits') {
4249                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4250             } elsif ($opt eq 'cpulimit') {
4251                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4252             } else {
4253                 die "skip\n";
4254             }
4255         };
4256         if (my $err = $@) {
4257             &$add_error($opt, $err) if $err ne "skip\n";
4258         } else {
4259             # save new config if hotplug was successful
4260             delete $conf->{$opt};
4261             vmconfig_undelete_pending_option($conf, $opt);
4262             PVE::QemuConfig->write_config($vmid, $conf);
4263             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4264         }
4265     }
4266
4267     foreach my $opt (keys %{$conf->{pending}}) {
4268         next if $selection && !$selection->{$opt};
4269         my $value = $conf->{pending}->{$opt};
4270         eval {
4271             if ($opt eq 'hotplug') {
4272                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4273             } elsif ($opt eq 'tablet') {
4274                 die "skip\n" if !$hotplug_features->{usb};
4275                 if ($value == 1) {
4276                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4277                 } elsif ($value == 0) {
4278                     vm_deviceunplug($vmid, $conf, $opt);
4279                 }
4280             } elsif ($opt =~ m/^usb\d+$/) {
4281                 die "skip\n";
4282                 # since we cannot reliably hot unplug usb devices
4283                 # we are disabling it
4284                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4285                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4286                 die "skip\n" if !$d;
4287                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4288             } elsif ($opt eq 'vcpus') {
4289                 die "skip\n" if !$hotplug_features->{cpu};
4290                 qemu_cpu_hotplug($vmid, $conf, $value);
4291             } elsif ($opt eq 'balloon') {
4292                 # enable/disable balloning device is not hotpluggable
4293                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4294                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4295                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4296
4297                 # allow manual ballooning if shares is set to zero
4298                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4299                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4300                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4301                 }
4302             } elsif ($opt =~ m/^net(\d+)$/) {
4303                 # some changes can be done without hotplug
4304                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4305                                     $vmid, $opt, $value);
4306             } elsif (is_valid_drivename($opt)) {
4307                 # some changes can be done without hotplug
4308                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4309                                      $vmid, $opt, $value, 1);
4310             } elsif ($opt =~ m/^memory$/) { #dimms
4311                 die "skip\n" if !$hotplug_features->{memory};
4312                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4313             } elsif ($opt eq 'cpuunits') {
4314                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4315             } elsif ($opt eq 'cpulimit') {
4316                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4317                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4318             } else {
4319                 die "skip\n";  # skip non-hot-pluggable options
4320             }
4321         };
4322         if (my $err = $@) {
4323             &$add_error($opt, $err) if $err ne "skip\n";
4324         } else {
4325             # save new config if hotplug was successful
4326             $conf->{$opt} = $value;
4327             delete $conf->{pending}->{$opt};
4328             PVE::QemuConfig->write_config($vmid, $conf);
4329             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4330         }
4331     }
4332 }
4333
4334 sub try_deallocate_drive {
4335     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4336
4337     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4338         my $volid = $drive->{file};
4339         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4340             my $sid = PVE::Storage::parse_volume_id($volid);
4341             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4342
4343             # check if the disk is really unused
4344             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4345                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4346             PVE::Storage::vdisk_free($storecfg, $volid);
4347             return 1;
4348         } else {
4349             # If vm is not owner of this disk remove from config
4350             return 1;
4351         }
4352     }
4353
4354     return undef;
4355 }
4356
4357 sub vmconfig_delete_or_detach_drive {
4358     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4359
4360     my $drive = parse_drive($opt, $conf->{$opt});
4361
4362     my $rpcenv = PVE::RPCEnvironment::get();
4363     my $authuser = $rpcenv->get_user();
4364
4365     if ($force) {
4366         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4367         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4368     } else {
4369         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4370     }
4371 }
4372
4373 sub vmconfig_apply_pending {
4374     my ($vmid, $conf, $storecfg) = @_;
4375
4376     # cold plug
4377
4378     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4379     while (my ($opt, $force) = each %$pending_delete_hash) {
4380         die "internal error" if $opt =~ m/^unused/;
4381         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4382         if (!defined($conf->{$opt})) {
4383             vmconfig_undelete_pending_option($conf, $opt);
4384             PVE::QemuConfig->write_config($vmid, $conf);
4385         } elsif (is_valid_drivename($opt)) {
4386             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4387             vmconfig_undelete_pending_option($conf, $opt);
4388             delete $conf->{$opt};
4389             PVE::QemuConfig->write_config($vmid, $conf);
4390         } else {
4391             vmconfig_undelete_pending_option($conf, $opt);
4392             delete $conf->{$opt};
4393             PVE::QemuConfig->write_config($vmid, $conf);
4394         }
4395     }
4396
4397     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4398
4399     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4400         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4401
4402         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4403             # skip if nothing changed
4404         } elsif (is_valid_drivename($opt)) {
4405             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4406                 if defined($conf->{$opt});
4407             $conf->{$opt} = $conf->{pending}->{$opt};
4408         } else {
4409             $conf->{$opt} = $conf->{pending}->{$opt};
4410         }
4411
4412         delete $conf->{pending}->{$opt};
4413         PVE::QemuConfig->write_config($vmid, $conf);
4414     }
4415 }
4416
4417 my $safe_num_ne = sub {
4418     my ($a, $b) = @_;
4419
4420     return 0 if !defined($a) && !defined($b);
4421     return 1 if !defined($a);
4422     return 1 if !defined($b);
4423
4424     return $a != $b;
4425 };
4426
4427 my $safe_string_ne = sub {
4428     my ($a, $b) = @_;
4429
4430     return 0 if !defined($a) && !defined($b);
4431     return 1 if !defined($a);
4432     return 1 if !defined($b);
4433
4434     return $a ne $b;
4435 };
4436
4437 sub vmconfig_update_net {
4438     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4439
4440     my $newnet = parse_net($value);
4441
4442     if ($conf->{$opt}) {
4443         my $oldnet = parse_net($conf->{$opt});
4444
4445         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4446             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4447             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4448             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4449
4450             # for non online change, we try to hot-unplug
4451             die "skip\n" if !$hotplug;
4452             vm_deviceunplug($vmid, $conf, $opt);
4453         } else {
4454
4455             die "internal error" if $opt !~ m/net(\d+)/;
4456             my $iface = "tap${vmid}i$1";
4457
4458             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4459                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4460                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4461                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4462                 PVE::Network::tap_unplug($iface);
4463                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4464             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4465                 # Rate can be applied on its own but any change above needs to
4466                 # include the rate in tap_plug since OVS resets everything.
4467                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4468             }
4469
4470             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4471                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4472             }
4473
4474             return 1;
4475         }
4476     }
4477
4478     if ($hotplug) {
4479         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4480     } else {
4481         die "skip\n";
4482     }
4483 }
4484
4485 sub vmconfig_update_disk {
4486     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4487
4488     # fixme: do we need force?
4489
4490     my $drive = parse_drive($opt, $value);
4491
4492     if ($conf->{$opt}) {
4493
4494         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4495
4496             my $media = $drive->{media} || 'disk';
4497             my $oldmedia = $old_drive->{media} || 'disk';
4498             die "unable to change media type\n" if $media ne $oldmedia;
4499
4500             if (!drive_is_cdrom($old_drive)) {
4501
4502                 if ($drive->{file} ne $old_drive->{file}) {
4503
4504                     die "skip\n" if !$hotplug;
4505
4506                     # unplug and register as unused
4507                     vm_deviceunplug($vmid, $conf, $opt);
4508                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4509
4510                 } else {
4511                     # update existing disk
4512
4513                     # skip non hotpluggable value
4514                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4515                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4516                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4517                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4518                         die "skip\n";
4519                     }
4520
4521                     # apply throttle
4522                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4523                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4524                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4525                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4526                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4527                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4528                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4529                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4530                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4531                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4532                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4533                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4534                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4535                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4536                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4537                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4538                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4539                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4540
4541                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4542                                                    ($drive->{mbps} || 0)*1024*1024,
4543                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4544                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4545                                                    $drive->{iops} || 0,
4546                                                    $drive->{iops_rd} || 0,
4547                                                    $drive->{iops_wr} || 0,
4548                                                    ($drive->{mbps_max} || 0)*1024*1024,
4549                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4550                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4551                                                    $drive->{iops_max} || 0,
4552                                                    $drive->{iops_rd_max} || 0,
4553                                                    $drive->{iops_wr_max} || 0,
4554                                                    $drive->{bps_max_length} || 1,
4555                                                    $drive->{bps_rd_max_length} || 1,
4556                                                    $drive->{bps_wr_max_length} || 1,
4557                                                    $drive->{iops_max_length} || 1,
4558                                                    $drive->{iops_rd_max_length} || 1,
4559                                                    $drive->{iops_wr_max_length} || 1);
4560
4561                     }
4562
4563                     return 1;
4564                 }
4565
4566             } else { # cdrom
4567
4568                 if ($drive->{file} eq 'none') {
4569                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4570                 } else {
4571                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4572                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4573                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4574                 }
4575
4576                 return 1;
4577             }
4578         }
4579     }
4580
4581     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4582     # hotplug new disks
4583     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4584     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4585 }
4586
4587 sub vm_start {
4588     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4589         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4590
4591     PVE::QemuConfig->lock_config($vmid, sub {
4592         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4593
4594         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4595
4596         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4597
4598         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4599
4600         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4601             vmconfig_apply_pending($vmid, $conf, $storecfg);
4602             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4603         }
4604
4605         my $defaults = load_defaults();
4606
4607         # set environment variable useful inside network script
4608         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4609
4610         my $local_volumes = {};
4611
4612         if ($targetstorage) {
4613             foreach_drive($conf, sub {
4614                 my ($ds, $drive) = @_;
4615
4616                 return if drive_is_cdrom($drive);
4617
4618                 my $volid = $drive->{file};
4619
4620                 return if !$volid;
4621
4622                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4623
4624                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4625                 return if $scfg->{shared};
4626                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4627             });
4628
4629             my $format = undef;
4630
4631             foreach my $opt (sort keys %$local_volumes) {
4632
4633                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4634                 my $drive = parse_drive($opt, $conf->{$opt});
4635
4636                 #if remote storage is specified, use default format
4637                 if ($targetstorage && $targetstorage ne "1") {
4638                     $storeid = $targetstorage;
4639                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4640                     $format = $defFormat;
4641                 } else {
4642                     #else we use same format than original
4643                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4644                     $format = qemu_img_format($scfg, $volid);
4645                 }
4646
4647                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4648                 my $newdrive = $drive;
4649                 $newdrive->{format} = $format;
4650                 $newdrive->{file} = $newvolid;
4651                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4652                 $local_volumes->{$opt} = $drivestr;
4653                 #pass drive to conf for command line
4654                 $conf->{$opt} = $drivestr;
4655             }
4656         }
4657
4658         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4659
4660         my $migrate_port = 0;
4661         my $migrate_uri;
4662         if ($statefile) {
4663             if ($statefile eq 'tcp') {
4664                 my $localip = "localhost";
4665                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4666                 my $nodename = PVE::INotify::nodename();
4667
4668                 if (!defined($migration_type)) {
4669                     if (defined($datacenterconf->{migration}->{type})) {
4670                         $migration_type = $datacenterconf->{migration}->{type};
4671                     } else {
4672                         $migration_type = 'secure';
4673                     }
4674                 }
4675
4676                 if ($migration_type eq 'insecure') {
4677                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4678                     if ($migrate_network_addr) {
4679                         $localip = $migrate_network_addr;
4680                     } else {
4681                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4682                     }
4683
4684                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4685                 }
4686
4687                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4688                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4689                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4690                 push @$cmd, '-incoming', $migrate_uri;
4691                 push @$cmd, '-S';
4692
4693             } elsif ($statefile eq 'unix') {
4694                 # should be default for secure migrations as a ssh TCP forward
4695                 # tunnel is not deterministic reliable ready and fails regurarly
4696                 # to set up in time, so use UNIX socket forwards
4697                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4698                 unlink $socket_addr;
4699
4700                 $migrate_uri = "unix:$socket_addr";
4701
4702                 push @$cmd, '-incoming', $migrate_uri;
4703                 push @$cmd, '-S';
4704
4705             } else {
4706                 push @$cmd, '-loadstate', $statefile;
4707             }
4708         } elsif ($paused) {
4709             push @$cmd, '-S';
4710         }
4711
4712         # host pci devices
4713         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4714           my $d = parse_hostpci($conf->{"hostpci$i"});
4715           next if !$d;
4716           my $pcidevices = $d->{pciid};
4717           foreach my $pcidevice (@$pcidevices) {
4718                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4719
4720                 my $info = pci_device_info("0000:$pciid");
4721                 die "IOMMU not present\n" if !check_iommu_support();
4722                 die "no pci device info for device '$pciid'\n" if !$info;
4723                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4724                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4725           }
4726         }
4727
4728         PVE::Storage::activate_volumes($storecfg, $vollist);
4729
4730         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4731             my $cmd = [];
4732             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4733             eval  { run_command($cmd); };
4734         }
4735
4736         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4737                                                   : $defaults->{cpuunits};
4738
4739         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4740         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4741
4742         my %properties = (
4743             Slice => 'qemu.slice',
4744             KillMode => 'none',
4745             CPUShares => $cpuunits
4746         );
4747
4748         if (my $cpulimit = $conf->{cpulimit}) {
4749             $properties{CPUQuota} = int($cpulimit * 100);
4750         }
4751         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4752
4753         if ($conf->{hugepages}) {
4754
4755             my $code = sub {
4756                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4757                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4758
4759                 PVE::QemuServer::Memory::hugepages_mount();
4760                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4761
4762                 eval  {
4763                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4764                     run_command($cmd, %run_params);
4765                 };
4766
4767                 if (my $err = $@) {
4768                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4769                     die $err;
4770                 }
4771
4772                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4773             };
4774             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4775
4776         } else {
4777             eval  {
4778                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4779                 run_command($cmd, %run_params);
4780             };
4781         }
4782
4783         if (my $err = $@) {
4784             # deactivate volumes if start fails
4785             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4786             die "start failed: $err";
4787         }
4788
4789         print "migration listens on $migrate_uri\n" if $migrate_uri;
4790
4791         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4792             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4793             warn $@ if $@;
4794         }
4795
4796         #start nbd server for storage migration
4797         if ($targetstorage) {
4798             my $nodename = PVE::INotify::nodename();
4799             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4800             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4801             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4802             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4803
4804             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4805
4806             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4807
4808             foreach my $opt (sort keys %$local_volumes) {
4809                 my $volid = $local_volumes->{$opt};
4810                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4811                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4812                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4813             }
4814         }
4815
4816         if ($migratedfrom) {
4817             eval {
4818                 set_migration_caps($vmid);
4819             };
4820             warn $@ if $@;
4821
4822             if ($spice_port) {
4823                 print "spice listens on port $spice_port\n";
4824                 if ($spice_ticket) {
4825                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4826                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4827                 }
4828             }
4829
4830         } else {
4831             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4832                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4833                     if $conf->{balloon};
4834             }
4835
4836             foreach my $opt (keys %$conf) {
4837                 next if $opt !~  m/^net\d+$/;
4838                 my $nicconf = parse_net($conf->{$opt});
4839                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4840             }
4841         }
4842
4843         vm_mon_cmd_nocheck($vmid, 'qom-set',
4844                     path => "machine/peripheral/balloon0",
4845                     property => "guest-stats-polling-interval",
4846                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4847
4848     });
4849 }
4850
4851 sub vm_mon_cmd {
4852     my ($vmid, $execute, %params) = @_;
4853
4854     my $cmd = { execute => $execute, arguments => \%params };
4855     vm_qmp_command($vmid, $cmd);
4856 }
4857
4858 sub vm_mon_cmd_nocheck {
4859     my ($vmid, $execute, %params) = @_;
4860
4861     my $cmd = { execute => $execute, arguments => \%params };
4862     vm_qmp_command($vmid, $cmd, 1);
4863 }
4864
4865 sub vm_qmp_command {
4866     my ($vmid, $cmd, $nocheck) = @_;
4867
4868     my $res;
4869
4870     my $timeout;
4871     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4872         $timeout = $cmd->{arguments}->{timeout};
4873         delete $cmd->{arguments}->{timeout};
4874     }
4875
4876     eval {
4877         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4878         my $sname = qmp_socket($vmid);
4879         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4880             my $qmpclient = PVE::QMPClient->new();
4881
4882             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4883         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4884             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4885                 if scalar(%{$cmd->{arguments}});
4886             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4887         } else {
4888             die "unable to open monitor socket\n";
4889         }
4890     };
4891     if (my $err = $@) {
4892         syslog("err", "VM $vmid qmp command failed - $err");
4893         die $err;
4894     }
4895
4896     return $res;
4897 }
4898
4899 sub vm_human_monitor_command {
4900     my ($vmid, $cmdline) = @_;
4901
4902     my $res;
4903
4904     my $cmd = {
4905         execute => 'human-monitor-command',
4906         arguments => { 'command-line' => $cmdline},
4907     };
4908
4909     return vm_qmp_command($vmid, $cmd);
4910 }
4911
4912 sub vm_commandline {
4913     my ($storecfg, $vmid) = @_;
4914
4915     my $conf = PVE::QemuConfig->load_config($vmid);
4916
4917     my $defaults = load_defaults();
4918
4919     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4920
4921     return PVE::Tools::cmd2string($cmd);
4922 }
4923
4924 sub vm_reset {
4925     my ($vmid, $skiplock) = @_;
4926
4927     PVE::QemuConfig->lock_config($vmid, sub {
4928
4929         my $conf = PVE::QemuConfig->load_config($vmid);
4930
4931         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4932
4933         vm_mon_cmd($vmid, "system_reset");
4934     });
4935 }
4936
4937 sub get_vm_volumes {
4938     my ($conf) = @_;
4939
4940     my $vollist = [];
4941     foreach_volid($conf, sub {
4942         my ($volid, $attr) = @_;
4943
4944         return if $volid =~ m|^/|;
4945
4946         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4947         return if !$sid;
4948
4949         push @$vollist, $volid;
4950     });
4951
4952     return $vollist;
4953 }
4954
4955 sub vm_stop_cleanup {
4956     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4957
4958     eval {
4959
4960         if (!$keepActive) {
4961             my $vollist = get_vm_volumes($conf);
4962             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4963         }
4964
4965         foreach my $ext (qw(mon qmp pid vnc qga)) {
4966             unlink "/var/run/qemu-server/${vmid}.$ext";
4967         }
4968
4969         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4970     };
4971     warn $@ if $@; # avoid errors - just warn
4972 }
4973
4974 # Note: use $nockeck to skip tests if VM configuration file exists.
4975 # We need that when migration VMs to other nodes (files already moved)
4976 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4977 sub vm_stop {
4978     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4979
4980     $force = 1 if !defined($force) && !$shutdown;
4981
4982     if ($migratedfrom){
4983         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4984         kill 15, $pid if $pid;
4985         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4986         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4987         return;
4988     }
4989
4990     PVE::QemuConfig->lock_config($vmid, sub {
4991
4992         my $pid = check_running($vmid, $nocheck);
4993         return if !$pid;
4994
4995         my $conf;
4996         if (!$nocheck) {
4997             $conf = PVE::QemuConfig->load_config($vmid);
4998             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4999             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5000                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5001                 $timeout = $opts->{down} if $opts->{down};
5002             }
5003         }
5004
5005         $timeout = 60 if !defined($timeout);
5006
5007         eval {
5008             if ($shutdown) {
5009                 if (defined($conf) && $conf->{agent}) {
5010                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5011                 } else {
5012                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5013                 }
5014             } else {
5015                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5016             }
5017         };
5018         my $err = $@;
5019
5020         if (!$err) {
5021             my $count = 0;
5022             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5023                 $count++;
5024                 sleep 1;
5025             }
5026
5027             if ($count >= $timeout) {
5028                 if ($force) {
5029                     warn "VM still running - terminating now with SIGTERM\n";
5030                     kill 15, $pid;
5031                 } else {
5032                     die "VM quit/powerdown failed - got timeout\n";
5033                 }
5034             } else {
5035                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5036                 return;
5037             }
5038         } else {
5039             if ($force) {
5040                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5041                 kill 15, $pid;
5042             } else {
5043                 die "VM quit/powerdown failed\n";
5044             }
5045         }
5046
5047         # wait again
5048         $timeout = 10;
5049
5050         my $count = 0;
5051         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5052             $count++;
5053             sleep 1;
5054         }
5055
5056         if ($count >= $timeout) {
5057             warn "VM still running - terminating now with SIGKILL\n";
5058             kill 9, $pid;
5059             sleep 1;
5060         }
5061
5062         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5063    });
5064 }
5065
5066 sub vm_suspend {
5067     my ($vmid, $skiplock) = @_;
5068
5069     PVE::QemuConfig->lock_config($vmid, sub {
5070
5071         my $conf = PVE::QemuConfig->load_config($vmid);
5072
5073         PVE::QemuConfig->check_lock($conf)
5074             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5075
5076         vm_mon_cmd($vmid, "stop");
5077     });
5078 }
5079
5080 sub vm_resume {
5081     my ($vmid, $skiplock, $nocheck) = @_;
5082
5083     PVE::QemuConfig->lock_config($vmid, sub {
5084
5085         if (!$nocheck) {
5086
5087             my $conf = PVE::QemuConfig->load_config($vmid);
5088
5089             PVE::QemuConfig->check_lock($conf)
5090                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5091
5092             vm_mon_cmd($vmid, "cont");
5093
5094         } else {
5095             vm_mon_cmd_nocheck($vmid, "cont");
5096         }
5097     });
5098 }
5099
5100 sub vm_sendkey {
5101     my ($vmid, $skiplock, $key) = @_;
5102
5103     PVE::QemuConfig->lock_config($vmid, sub {
5104
5105         my $conf = PVE::QemuConfig->load_config($vmid);
5106
5107         # there is no qmp command, so we use the human monitor command
5108         vm_human_monitor_command($vmid, "sendkey $key");
5109     });
5110 }
5111
5112 sub vm_destroy {
5113     my ($storecfg, $vmid, $skiplock) = @_;
5114
5115     PVE::QemuConfig->lock_config($vmid, sub {
5116
5117         my $conf = PVE::QemuConfig->load_config($vmid);
5118
5119         if (!check_running($vmid)) {
5120             destroy_vm($storecfg, $vmid, undef, $skiplock);
5121         } else {
5122             die "VM $vmid is running - destroy failed\n";
5123         }
5124     });
5125 }
5126
5127 # pci helpers
5128
5129 sub file_write {
5130     my ($filename, $buf) = @_;
5131
5132     my $fh = IO::File->new($filename, "w");
5133     return undef if !$fh;
5134
5135     my $res = print $fh $buf;
5136
5137     $fh->close();
5138
5139     return $res;
5140 }
5141
5142 sub pci_device_info {
5143     my ($name) = @_;
5144
5145     my $res;
5146
5147     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5148     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5149
5150     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5151     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5152
5153     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5154     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5155
5156     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5157     return undef if !defined($product) || $product !~ s/^0x//;
5158
5159     $res = {
5160         name => $name,
5161         vendor => $vendor,
5162         product => $product,
5163         domain => $domain,
5164         bus => $bus,
5165         slot => $slot,
5166         func => $func,
5167         irq => $irq,
5168         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5169     };
5170
5171     return $res;
5172 }
5173
5174 sub pci_dev_reset {
5175     my ($dev) = @_;
5176
5177     my $name = $dev->{name};
5178
5179     my $fn = "$pcisysfs/devices/$name/reset";
5180
5181     return file_write($fn, "1");
5182 }
5183
5184 sub pci_dev_bind_to_vfio {
5185     my ($dev) = @_;
5186
5187     my $name = $dev->{name};
5188
5189     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5190
5191     if (!-d $vfio_basedir) {
5192         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5193     }
5194     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5195
5196     my $testdir = "$vfio_basedir/$name";
5197     return 1 if -d $testdir;
5198
5199     my $data = "$dev->{vendor} $dev->{product}";
5200     return undef if !file_write("$vfio_basedir/new_id", $data);
5201
5202     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5203     if (!file_write($fn, $name)) {
5204         return undef if -f $fn;
5205     }
5206
5207     $fn = "$vfio_basedir/bind";
5208     if (! -d $testdir) {
5209         return undef if !file_write($fn, $name);
5210     }
5211
5212     return -d $testdir;
5213 }
5214
5215 sub pci_dev_group_bind_to_vfio {
5216     my ($pciid) = @_;
5217
5218     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5219
5220     if (!-d $vfio_basedir) {
5221         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5222     }
5223     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5224
5225     # get IOMMU group devices
5226     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5227       my @devs = grep /^0000:/, readdir($D);
5228     closedir($D);
5229
5230     foreach my $pciid (@devs) {
5231         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5232
5233         # pci bridges, switches or root ports are not supported
5234         # they have a pci_bus subdirectory so skip them
5235         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5236
5237         my $info = pci_device_info($1);
5238         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5239     }
5240
5241     return 1;
5242 }
5243
5244 # vzdump restore implementaion
5245
5246 sub tar_archive_read_firstfile {
5247     my $archive = shift;
5248
5249     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5250
5251     # try to detect archive type first
5252     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5253         die "unable to open file '$archive'\n";
5254     my $firstfile = <$fh>;
5255     kill 15, $pid;
5256     close $fh;
5257
5258     die "ERROR: archive contaions no data\n" if !$firstfile;
5259     chomp $firstfile;
5260
5261     return $firstfile;
5262 }
5263
5264 sub tar_restore_cleanup {
5265     my ($storecfg, $statfile) = @_;
5266
5267     print STDERR "starting cleanup\n";
5268
5269     if (my $fd = IO::File->new($statfile, "r")) {
5270         while (defined(my $line = <$fd>)) {
5271             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5272                 my $volid = $2;
5273                 eval {
5274                     if ($volid =~ m|^/|) {
5275                         unlink $volid || die 'unlink failed\n';
5276                     } else {
5277                         PVE::Storage::vdisk_free($storecfg, $volid);
5278                     }
5279                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5280                 };
5281                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5282             } else {
5283                 print STDERR "unable to parse line in statfile - $line";
5284             }
5285         }
5286         $fd->close();
5287     }
5288 }
5289
5290 sub restore_archive {
5291     my ($archive, $vmid, $user, $opts) = @_;
5292
5293     my $format = $opts->{format};
5294     my $comp;
5295
5296     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5297         $format = 'tar' if !$format;
5298         $comp = 'gzip';
5299     } elsif ($archive =~ m/\.tar$/) {
5300         $format = 'tar' if !$format;
5301     } elsif ($archive =~ m/.tar.lzo$/) {
5302         $format = 'tar' if !$format;
5303         $comp = 'lzop';
5304     } elsif ($archive =~ m/\.vma$/) {
5305         $format = 'vma' if !$format;
5306     } elsif ($archive =~ m/\.vma\.gz$/) {
5307         $format = 'vma' if !$format;
5308         $comp = 'gzip';
5309     } elsif ($archive =~ m/\.vma\.lzo$/) {
5310         $format = 'vma' if !$format;
5311         $comp = 'lzop';
5312     } else {
5313         $format = 'vma' if !$format; # default
5314     }
5315
5316     # try to detect archive format
5317     if ($format eq 'tar') {
5318         return restore_tar_archive($archive, $vmid, $user, $opts);
5319     } else {
5320         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5321     }
5322 }
5323
5324 sub restore_update_config_line {
5325     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5326
5327     return if $line =~ m/^\#qmdump\#/;
5328     return if $line =~ m/^\#vzdump\#/;
5329     return if $line =~ m/^lock:/;
5330     return if $line =~ m/^unused\d+:/;
5331     return if $line =~ m/^parent:/;
5332     return if $line =~ m/^template:/; # restored VM is never a template
5333
5334     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5335     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5336         # try to convert old 1.X settings
5337         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5338         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5339             my ($model, $macaddr) = split(/\=/, $devconfig);
5340             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5341             my $net = {
5342                 model => $model,
5343                 bridge => "vmbr$ind",
5344                 macaddr => $macaddr,
5345             };
5346             my $netstr = print_net($net);
5347
5348             print $outfd "net$cookie->{netcount}: $netstr\n";
5349             $cookie->{netcount}++;
5350         }
5351     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5352         my ($id, $netstr) = ($1, $2);
5353         my $net = parse_net($netstr);
5354         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5355         $netstr = print_net($net);
5356         print $outfd "$id: $netstr\n";
5357     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5358         my $virtdev = $1;
5359         my $value = $3;
5360         my $di = parse_drive($virtdev, $value);
5361         if (defined($di->{backup}) && !$di->{backup}) {
5362             print $outfd "#$line";
5363         } elsif ($map->{$virtdev}) {
5364             delete $di->{format}; # format can change on restore
5365             $di->{file} = $map->{$virtdev};
5366             $value = print_drive($vmid, $di);
5367             print $outfd "$virtdev: $value\n";
5368         } else {
5369             print $outfd $line;
5370         }
5371     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5372         my ($uuid, $uuid_str);
5373         UUID::generate($uuid);
5374         UUID::unparse($uuid, $uuid_str);
5375         my $smbios1 = parse_smbios1($2);
5376         $smbios1->{uuid} = $uuid_str;
5377         print $outfd $1.print_smbios1($smbios1)."\n";
5378     } else {
5379         print $outfd $line;
5380     }
5381 }
5382
5383 sub scan_volids {
5384     my ($cfg, $vmid) = @_;
5385
5386     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5387
5388     my $volid_hash = {};
5389     foreach my $storeid (keys %$info) {
5390         foreach my $item (@{$info->{$storeid}}) {
5391             next if !($item->{volid} && $item->{size});
5392             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5393             $volid_hash->{$item->{volid}} = $item;
5394         }
5395     }
5396
5397     return $volid_hash;
5398 }
5399
5400 sub is_volume_in_use {
5401     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5402
5403     my $path = PVE::Storage::path($storecfg, $volid);
5404
5405     my $scan_config = sub {
5406         my ($cref, $snapname) = @_;
5407
5408         foreach my $key (keys %$cref) {
5409             my $value = $cref->{$key};
5410             if (is_valid_drivename($key)) {
5411                 next if $skip_drive && $key eq $skip_drive;
5412                 my $drive = parse_drive($key, $value);
5413                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5414                 return 1 if $volid eq $drive->{file};
5415                 if ($drive->{file} =~ m!^/!) {
5416                     return 1 if $drive->{file} eq $path;
5417                 } else {
5418                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5419                     next if !$storeid;
5420                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5421                     next if !$scfg;
5422                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5423                 }
5424             }
5425         }
5426
5427         return 0;
5428     };
5429
5430     return 1 if &$scan_config($conf);
5431
5432     undef $skip_drive;
5433
5434     foreach my $snapname (keys %{$conf->{snapshots}}) {
5435         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5436     }
5437
5438     return 0;
5439 }
5440
5441 sub update_disksize {
5442     my ($vmid, $conf, $volid_hash) = @_;
5443
5444     my $changes;
5445
5446     # used and unused disks
5447     my $referenced = {};
5448
5449     # Note: it is allowed to define multiple storages with same path (alias), so
5450     # we need to check both 'volid' and real 'path' (two different volid can point
5451     # to the same path).
5452
5453     my $referencedpath = {};
5454
5455     # update size info
5456     foreach my $opt (keys %$conf) {
5457         if (is_valid_drivename($opt)) {
5458             my $drive = parse_drive($opt, $conf->{$opt});
5459             my $volid = $drive->{file};
5460             next if !$volid;
5461
5462             $referenced->{$volid} = 1;
5463             if ($volid_hash->{$volid} &&
5464                 (my $path = $volid_hash->{$volid}->{path})) {
5465                 $referencedpath->{$path} = 1;
5466             }
5467
5468             next if drive_is_cdrom($drive);
5469             next if !$volid_hash->{$volid};
5470
5471             $drive->{size} = $volid_hash->{$volid}->{size};
5472             my $new = print_drive($vmid, $drive);
5473             if ($new ne $conf->{$opt}) {
5474                 $changes = 1;
5475                 $conf->{$opt} = $new;
5476             }
5477         }
5478     }
5479
5480     # remove 'unusedX' entry if volume is used
5481     foreach my $opt (keys %$conf) {
5482         next if $opt !~ m/^unused\d+$/;
5483         my $volid = $conf->{$opt};
5484         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5485         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5486             $changes = 1;
5487             delete $conf->{$opt};
5488         }
5489
5490         $referenced->{$volid} = 1;
5491         $referencedpath->{$path} = 1 if $path;
5492     }
5493
5494     foreach my $volid (sort keys %$volid_hash) {
5495         next if $volid =~ m/vm-$vmid-state-/;
5496         next if $referenced->{$volid};
5497         my $path = $volid_hash->{$volid}->{path};
5498         next if !$path; # just to be sure
5499         next if $referencedpath->{$path};
5500         $changes = 1;
5501         PVE::QemuConfig->add_unused_volume($conf, $volid);
5502         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5503     }
5504
5505     return $changes;
5506 }
5507
5508 sub rescan {
5509     my ($vmid, $nolock) = @_;
5510
5511     my $cfg = PVE::Storage::config();
5512
5513     my $volid_hash = scan_volids($cfg, $vmid);
5514
5515     my $updatefn =  sub {
5516         my ($vmid) = @_;
5517
5518         my $conf = PVE::QemuConfig->load_config($vmid);
5519
5520         PVE::QemuConfig->check_lock($conf);
5521
5522         my $vm_volids = {};
5523         foreach my $volid (keys %$volid_hash) {
5524             my $info = $volid_hash->{$volid};
5525             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5526         }
5527
5528         my $changes = update_disksize($vmid, $conf, $vm_volids);
5529
5530         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5531     };
5532
5533     if (defined($vmid)) {
5534         if ($nolock) {
5535             &$updatefn($vmid);
5536         } else {
5537             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5538         }
5539     } else {
5540         my $vmlist = config_list();
5541         foreach my $vmid (keys %$vmlist) {
5542             if ($nolock) {
5543                 &$updatefn($vmid);
5544             } else {
5545                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5546             }
5547         }
5548     }
5549 }
5550
5551 sub restore_vma_archive {
5552     my ($archive, $vmid, $user, $opts, $comp) = @_;
5553
5554     my $input = $archive eq '-' ? "<&STDIN" : undef;
5555     my $readfrom = $archive;
5556
5557     my $uncomp = '';
5558     if ($comp) {
5559         $readfrom = '-';
5560         my $qarchive = PVE::Tools::shellquote($archive);
5561         if ($comp eq 'gzip') {
5562             $uncomp = "zcat $qarchive|";
5563         } elsif ($comp eq 'lzop') {
5564             $uncomp = "lzop -d -c $qarchive|";
5565         } else {
5566             die "unknown compression method '$comp'\n";
5567         }
5568
5569     }
5570
5571     my $tmpdir = "/var/tmp/vzdumptmp$$";
5572     rmtree $tmpdir;
5573
5574     # disable interrupts (always do cleanups)
5575     local $SIG{INT} =
5576         local $SIG{TERM} =
5577         local $SIG{QUIT} =
5578         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5579
5580     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5581     POSIX::mkfifo($mapfifo, 0600);
5582     my $fifofh;
5583
5584     my $openfifo = sub {
5585         open($fifofh, '>', $mapfifo) || die $!;
5586     };
5587
5588     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5589
5590     my $oldtimeout;
5591     my $timeout = 5;
5592
5593     my $devinfo = {};
5594
5595     my $rpcenv = PVE::RPCEnvironment::get();
5596
5597     my $conffile = PVE::QemuConfig->config_file($vmid);
5598     my $tmpfn = "$conffile.$$.tmp";
5599
5600     # Note: $oldconf is undef if VM does not exists
5601     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5602     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5603
5604     my $print_devmap = sub {
5605         my $virtdev_hash = {};
5606
5607         my $cfgfn = "$tmpdir/qemu-server.conf";
5608
5609         # we can read the config - that is already extracted
5610         my $fh = IO::File->new($cfgfn, "r") ||
5611             "unable to read qemu-server.conf - $!\n";
5612
5613         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5614         if (-f $fwcfgfn) {
5615             my $pve_firewall_dir = '/etc/pve/firewall';
5616             mkdir $pve_firewall_dir; # make sure the dir exists
5617             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5618         }
5619
5620         while (defined(my $line = <$fh>)) {
5621             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5622                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5623                 die "archive does not contain data for drive '$virtdev'\n"
5624                     if !$devinfo->{$devname};
5625                 if (defined($opts->{storage})) {
5626                     $storeid = $opts->{storage} || 'local';
5627                 } elsif (!$storeid) {
5628                     $storeid = 'local';
5629                 }
5630                 $format = 'raw' if !$format;
5631                 $devinfo->{$devname}->{devname} = $devname;
5632                 $devinfo->{$devname}->{virtdev} = $virtdev;
5633                 $devinfo->{$devname}->{format} = $format;
5634                 $devinfo->{$devname}->{storeid} = $storeid;
5635
5636                 # check permission on storage
5637                 my $pool = $opts->{pool}; # todo: do we need that?
5638                 if ($user ne 'root@pam') {
5639                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5640                 }
5641
5642                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5643             }
5644         }
5645
5646         foreach my $devname (keys %$devinfo) {
5647             die "found no device mapping information for device '$devname'\n"
5648                 if !$devinfo->{$devname}->{virtdev};
5649         }
5650
5651         my $cfg = PVE::Storage::config();
5652
5653         # create empty/temp config
5654         if ($oldconf) {
5655             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5656             foreach_drive($oldconf, sub {
5657                 my ($ds, $drive) = @_;
5658
5659                 return if drive_is_cdrom($drive);
5660
5661                 my $volid = $drive->{file};
5662
5663                 return if !$volid || $volid =~ m|^/|;
5664
5665                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5666                 return if !$path || !$owner || ($owner != $vmid);
5667
5668                 # Note: only delete disk we want to restore
5669                 # other volumes will become unused
5670                 if ($virtdev_hash->{$ds}) {
5671                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5672                     if (my $err = $@) {
5673                         warn $err;
5674                     }
5675                 }
5676             });
5677
5678             # delete vmstate files
5679             # since after the restore we have no snapshots anymore
5680             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5681                 my $snap = $oldconf->{snapshots}->{$snapname};
5682                 if ($snap->{vmstate}) {
5683                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5684                     if (my $err = $@) {
5685                         warn $err;
5686                     }
5687                 }
5688             }
5689         }
5690
5691         my $map = {};
5692         foreach my $virtdev (sort keys %$virtdev_hash) {
5693             my $d = $virtdev_hash->{$virtdev};
5694             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5695             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5696
5697             # test if requested format is supported
5698             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5699             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5700             $d->{format} = $defFormat if !$supported;
5701
5702             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5703                                                   $d->{format}, undef, $alloc_size);
5704             print STDERR "new volume ID is '$volid'\n";
5705             $d->{volid} = $volid;
5706             my $path = PVE::Storage::path($cfg, $volid);
5707
5708             PVE::Storage::activate_volumes($cfg,[$volid]);
5709
5710             my $write_zeros = 1;
5711             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5712                 $write_zeros = 0;
5713             }
5714
5715             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5716
5717             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5718             $map->{$virtdev} = $volid;
5719         }
5720
5721         $fh->seek(0, 0) || die "seek failed - $!\n";
5722
5723         my $outfd = new IO::File ($tmpfn, "w") ||
5724             die "unable to write config for VM $vmid\n";
5725
5726         my $cookie = { netcount => 0 };
5727         while (defined(my $line = <$fh>)) {
5728             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5729         }
5730
5731         $fh->close();
5732         $outfd->close();
5733     };
5734
5735     eval {
5736         # enable interrupts
5737         local $SIG{INT} =
5738             local $SIG{TERM} =
5739             local $SIG{QUIT} =
5740             local $SIG{HUP} =
5741             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5742         local $SIG{ALRM} = sub { die "got timeout\n"; };
5743
5744         $oldtimeout = alarm($timeout);
5745
5746         my $parser = sub {
5747             my $line = shift;
5748
5749             print "$line\n";
5750
5751             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5752                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5753                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5754             } elsif ($line =~ m/^CTIME: /) {
5755                 # we correctly received the vma config, so we can disable
5756                 # the timeout now for disk allocation (set to 10 minutes, so
5757                 # that we always timeout if something goes wrong)
5758                 alarm(600);
5759                 &$print_devmap();
5760                 print $fifofh "done\n";
5761                 my $tmp = $oldtimeout || 0;
5762                 $oldtimeout = undef;
5763                 alarm($tmp);
5764                 close($fifofh);
5765             }
5766         };
5767
5768         print "restore vma archive: $cmd\n";
5769         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5770     };
5771     my $err = $@;
5772
5773     alarm($oldtimeout) if $oldtimeout;
5774
5775     my $vollist = [];
5776     foreach my $devname (keys %$devinfo) {
5777         my $volid = $devinfo->{$devname}->{volid};
5778         push @$vollist, $volid if $volid;
5779     }
5780
5781     my $cfg = PVE::Storage::config();
5782     PVE::Storage::deactivate_volumes($cfg, $vollist);
5783
5784     unlink $mapfifo;
5785
5786     if ($err) {
5787         rmtree $tmpdir;
5788         unlink $tmpfn;
5789
5790         foreach my $devname (keys %$devinfo) {
5791             my $volid = $devinfo->{$devname}->{volid};
5792             next if !$volid;
5793             eval {
5794                 if ($volid =~ m|^/|) {
5795                     unlink $volid || die 'unlink failed\n';
5796                 } else {
5797                     PVE::Storage::vdisk_free($cfg, $volid);
5798                 }
5799                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5800             };
5801             print STDERR "unable to cleanup '$volid' - $@" if $@;
5802         }
5803         die $err;
5804     }
5805
5806     rmtree $tmpdir;
5807
5808     rename($tmpfn, $conffile) ||
5809         die "unable to commit configuration file '$conffile'\n";
5810
5811     PVE::Cluster::cfs_update(); # make sure we read new file
5812
5813     eval { rescan($vmid, 1); };
5814     warn $@ if $@;
5815 }
5816
5817 sub restore_tar_archive {
5818     my ($archive, $vmid, $user, $opts) = @_;
5819
5820     if ($archive ne '-') {
5821         my $firstfile = tar_archive_read_firstfile($archive);
5822         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5823             if $firstfile ne 'qemu-server.conf';
5824     }
5825
5826     my $storecfg = PVE::Storage::config();
5827
5828     # destroy existing data - keep empty config
5829     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5830     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5831
5832     my $tocmd = "/usr/lib/qemu-server/qmextract";
5833
5834     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5835     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5836     $tocmd .= ' --prealloc' if $opts->{prealloc};
5837     $tocmd .= ' --info' if $opts->{info};
5838
5839     # tar option "xf" does not autodetect compression when read from STDIN,
5840     # so we pipe to zcat
5841     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5842         PVE::Tools::shellquote("--to-command=$tocmd");
5843
5844     my $tmpdir = "/var/tmp/vzdumptmp$$";
5845     mkpath $tmpdir;
5846
5847     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5848     local $ENV{VZDUMP_VMID} = $vmid;
5849     local $ENV{VZDUMP_USER} = $user;
5850
5851     my $conffile = PVE::QemuConfig->config_file($vmid);
5852     my $tmpfn = "$conffile.$$.tmp";
5853
5854     # disable interrupts (always do cleanups)
5855     local $SIG{INT} =
5856         local $SIG{TERM} =
5857         local $SIG{QUIT} =
5858         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5859
5860     eval {
5861         # enable interrupts
5862         local $SIG{INT} =
5863             local $SIG{TERM} =
5864             local $SIG{QUIT} =
5865             local $SIG{HUP} =
5866             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5867
5868         if ($archive eq '-') {
5869             print "extracting archive from STDIN\n";
5870             run_command($cmd, input => "<&STDIN");
5871         } else {
5872             print "extracting archive '$archive'\n";
5873             run_command($cmd);
5874         }
5875
5876         return if $opts->{info};
5877
5878         # read new mapping
5879         my $map = {};
5880         my $statfile = "$tmpdir/qmrestore.stat";
5881         if (my $fd = IO::File->new($statfile, "r")) {
5882             while (defined (my $line = <$fd>)) {
5883                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5884                     $map->{$1} = $2 if $1;
5885                 } else {
5886                     print STDERR "unable to parse line in statfile - $line\n";
5887                 }
5888             }
5889             $fd->close();
5890         }
5891
5892         my $confsrc = "$tmpdir/qemu-server.conf";
5893
5894         my $srcfd = new IO::File($confsrc, "r") ||
5895             die "unable to open file '$confsrc'\n";
5896
5897         my $outfd = new IO::File ($tmpfn, "w") ||
5898             die "unable to write config for VM $vmid\n";
5899
5900         my $cookie = { netcount => 0 };
5901         while (defined (my $line = <$srcfd>)) {
5902             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5903         }
5904
5905         $srcfd->close();
5906         $outfd->close();
5907     };
5908     my $err = $@;
5909
5910     if ($err) {
5911
5912         unlink $tmpfn;
5913
5914         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5915
5916         die $err;
5917     }
5918
5919     rmtree $tmpdir;
5920
5921     rename $tmpfn, $conffile ||
5922         die "unable to commit configuration file '$conffile'\n";
5923
5924     PVE::Cluster::cfs_update(); # make sure we read new file
5925
5926     eval { rescan($vmid, 1); };
5927     warn $@ if $@;
5928 };
5929
5930 sub foreach_storage_used_by_vm {
5931     my ($conf, $func) = @_;
5932
5933     my $sidhash = {};
5934
5935     foreach_drive($conf, sub {
5936         my ($ds, $drive) = @_;
5937         return if drive_is_cdrom($drive);
5938
5939         my $volid = $drive->{file};
5940
5941         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5942         $sidhash->{$sid} = $sid if $sid;
5943     });
5944
5945     foreach my $sid (sort keys %$sidhash) {
5946         &$func($sid);
5947     }
5948 }
5949
5950 sub do_snapshots_with_qemu {
5951     my ($storecfg, $volid) = @_;
5952
5953     my $storage_name = PVE::Storage::parse_volume_id($volid);
5954
5955     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5956         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5957         return 1;
5958     }
5959
5960     if ($volid =~ m/\.(qcow2|qed)$/){
5961         return 1;
5962     }
5963
5964     return undef;
5965 }
5966
5967 sub qga_check_running {
5968     my ($vmid) = @_;
5969
5970     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5971     if ($@) {
5972         warn "Qemu Guest Agent is not running - $@";
5973         return 0;
5974     }
5975     return 1;
5976 }
5977
5978 sub template_create {
5979     my ($vmid, $conf, $disk) = @_;
5980
5981     my $storecfg = PVE::Storage::config();
5982
5983     foreach_drive($conf, sub {
5984         my ($ds, $drive) = @_;
5985
5986         return if drive_is_cdrom($drive);
5987         return if $disk && $ds ne $disk;
5988
5989         my $volid = $drive->{file};
5990         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5991
5992         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5993         $drive->{file} = $voliddst;
5994         $conf->{$ds} = print_drive($vmid, $drive);
5995         PVE::QemuConfig->write_config($vmid, $conf);
5996     });
5997 }
5998
5999 sub qemu_img_convert {
6000     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6001
6002     my $storecfg = PVE::Storage::config();
6003     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6004     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6005
6006     if ($src_storeid && $dst_storeid) {
6007
6008         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6009
6010         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6011         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6012
6013         my $src_format = qemu_img_format($src_scfg, $src_volname);
6014         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6015
6016         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6017         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6018
6019         my $cmd = [];
6020         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6021         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6022         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6023         if ($is_zero_initialized) {
6024             push @$cmd, "zeroinit:$dst_path";
6025         } else {
6026             push @$cmd, $dst_path;
6027         }
6028
6029         my $parser = sub {
6030             my $line = shift;
6031             if($line =~ m/\((\S+)\/100\%\)/){
6032                 my $percent = $1;
6033                 my $transferred = int($size * $percent / 100);
6034                 my $remaining = $size - $transferred;
6035
6036                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6037             }
6038
6039         };
6040
6041         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6042         my $err = $@;
6043         die "copy failed: $err" if $err;
6044     }
6045 }
6046
6047 sub qemu_img_format {
6048     my ($scfg, $volname) = @_;
6049
6050     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6051         return $1;
6052     } else {
6053         return "raw";
6054     }
6055 }
6056
6057 sub qemu_drive_mirror {
6058     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6059
6060     $jobs = {} if !$jobs;
6061
6062     my $qemu_target;
6063     my $format;
6064     $jobs->{"drive-$drive"} = {};
6065
6066     if ($dst_volid =~ /^nbd:/) {
6067         $qemu_target = $dst_volid;
6068         $format = "nbd";
6069     } else {
6070         my $storecfg = PVE::Storage::config();
6071         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6072
6073         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6074
6075         $format = qemu_img_format($dst_scfg, $dst_volname);
6076
6077         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6078
6079         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6080     }
6081
6082     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6083     $opts->{format} = $format if $format;
6084
6085     print "drive mirror is starting for drive-$drive\n";
6086
6087     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6088
6089     if (my $err = $@) {
6090         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6091         die "mirroring error: $err";
6092     }
6093
6094     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6095 }
6096
6097 sub qemu_drive_mirror_monitor {
6098     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6099
6100     eval {
6101         my $err_complete = 0;
6102
6103         while (1) {
6104             die "storage migration timed out\n" if $err_complete > 300;
6105
6106             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6107
6108             my $running_mirror_jobs = {};
6109             foreach my $stat (@$stats) {
6110                 next if $stat->{type} ne 'mirror';
6111                 $running_mirror_jobs->{$stat->{device}} = $stat;
6112             }
6113
6114             my $readycounter = 0;
6115
6116             foreach my $job (keys %$jobs) {
6117
6118                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6119                     print "$job : finished\n";
6120                     delete $jobs->{$job};
6121                     next;
6122                 }
6123
6124                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6125
6126                 my $busy = $running_mirror_jobs->{$job}->{busy};
6127                 my $ready = $running_mirror_jobs->{$job}->{ready};
6128                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6129                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6130                     my $remaining = $total - $transferred;
6131                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6132
6133                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6134                 }
6135
6136                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6137             }
6138
6139             last if scalar(keys %$jobs) == 0;
6140
6141             if ($readycounter == scalar(keys %$jobs)) {
6142                 print "all mirroring jobs are ready \n";
6143                 last if $skipcomplete; #do the complete later
6144
6145                 if ($vmiddst && $vmiddst != $vmid) {
6146                     my $agent_running = $qga && qga_check_running($vmid);
6147                     if ($agent_running) {
6148                         print "freeze filesystem\n";
6149                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6150                     } else {
6151                         print "suspend vm\n";
6152                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6153                     }
6154
6155                     # if we clone a disk for a new target vm, we don't switch the disk
6156                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6157
6158                     if ($agent_running) {
6159                         print "unfreeze filesystem\n";
6160                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6161                     } else {
6162                         print "resume vm\n";
6163                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6164                     }
6165
6166                     last;
6167                 } else {
6168
6169                     foreach my $job (keys %$jobs) {
6170                         # try to switch the disk if source and destination are on the same guest
6171                         print "$job: Completing block job...\n";
6172
6173                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6174                         if ($@ =~ m/cannot be completed/) {
6175                             print "$job: Block job cannot be completed, try again.\n";
6176                             $err_complete++;
6177                         }else {
6178                             print "$job: Completed successfully.\n";
6179                             $jobs->{$job}->{complete} = 1;
6180                         }
6181                     }
6182                 }
6183             }
6184             sleep 1;
6185         }
6186     };
6187     my $err = $@;
6188
6189     if ($err) {
6190         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6191         die "mirroring error: $err";
6192     }
6193
6194 }
6195
6196 sub qemu_blockjobs_cancel {
6197     my ($vmid, $jobs) = @_;
6198
6199     foreach my $job (keys %$jobs) {
6200         print "$job: Cancelling block job\n";
6201         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6202         $jobs->{$job}->{cancel} = 1;
6203     }
6204
6205     while (1) {
6206         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6207
6208         my $running_jobs = {};
6209         foreach my $stat (@$stats) {
6210             $running_jobs->{$stat->{device}} = $stat;
6211         }
6212
6213         foreach my $job (keys %$jobs) {
6214
6215             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6216                 print "$job: Done.\n";
6217                 delete $jobs->{$job};
6218             }
6219         }
6220
6221         last if scalar(keys %$jobs) == 0;
6222
6223         sleep 1;
6224     }
6225 }
6226
6227 sub clone_disk {
6228     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6229         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6230
6231     my $newvolid;
6232
6233     if (!$full) {
6234         print "create linked clone of drive $drivename ($drive->{file})\n";
6235         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6236         push @$newvollist, $newvolid;
6237     } else {
6238
6239         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6240         $storeid = $storage if $storage;
6241
6242         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6243         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6244
6245         print "create full clone of drive $drivename ($drive->{file})\n";
6246         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, undef, ($size/1024));
6247         push @$newvollist, $newvolid;
6248
6249         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6250
6251         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6252         if (!$running || $snapname) {
6253             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6254         } else {
6255
6256             my $kvmver = get_running_qemu_version ($vmid);
6257             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6258                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6259                     if $drive->{iothread};
6260             }
6261
6262             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6263         }
6264     }
6265
6266     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6267
6268     my $disk = $drive;
6269     $disk->{format} = undef;
6270     $disk->{file} = $newvolid;
6271     $disk->{size} = $size;
6272
6273     return $disk;
6274 }
6275
6276 # this only works if VM is running
6277 sub get_current_qemu_machine {
6278     my ($vmid) = @_;
6279
6280     my $cmd = { execute => 'query-machines', arguments => {} };
6281     my $res = vm_qmp_command($vmid, $cmd);
6282
6283     my ($current, $default);
6284     foreach my $e (@$res) {
6285         $default = $e->{name} if $e->{'is-default'};
6286         $current = $e->{name} if $e->{'is-current'};
6287     }
6288
6289     # fallback to the default machine if current is not supported by qemu
6290     return $current || $default || 'pc';
6291 }
6292
6293 sub get_running_qemu_version {
6294     my ($vmid) = @_;
6295     my $cmd = { execute => 'query-version', arguments => {} };
6296     my $res = vm_qmp_command($vmid, $cmd);
6297     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6298 }
6299
6300 sub qemu_machine_feature_enabled {
6301     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6302
6303     my $current_major;
6304     my $current_minor;
6305
6306     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6307
6308         $current_major = $3;
6309         $current_minor = $4;
6310
6311     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6312
6313         $current_major = $1;
6314         $current_minor = $2;
6315     }
6316
6317     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6318
6319
6320 }
6321
6322 sub qemu_machine_pxe {
6323     my ($vmid, $conf, $machine) = @_;
6324
6325     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6326
6327     foreach my $opt (keys %$conf) {
6328         next if $opt !~ m/^net(\d+)$/;
6329         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6330         next if !$net;
6331         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6332         return $machine.".pxe" if $romfile =~ m/pxe/;
6333         last;
6334     }
6335
6336     return $machine;
6337 }
6338
6339 sub qemu_use_old_bios_files {
6340     my ($machine_type) = @_;
6341
6342     return if !$machine_type;
6343
6344     my $use_old_bios_files = undef;
6345
6346     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6347         $machine_type = $1;
6348         $use_old_bios_files = 1;
6349     } else {
6350         my $kvmver = kvm_user_version();
6351         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6352         # load new efi bios files on migration. So this hack is required to allow
6353         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6354         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6355         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6356     }
6357
6358     return ($use_old_bios_files, $machine_type);
6359 }
6360
6361 sub create_efidisk {
6362     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6363
6364     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6365
6366     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6367     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6368     PVE::Storage::activate_volumes($storecfg, [$volid]);
6369
6370     my $path = PVE::Storage::path($storecfg, $volid);
6371     eval {
6372         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6373     };
6374     die "Copying EFI vars image failed: $@" if $@;
6375
6376     return ($volid, $vars_size);
6377 }
6378
6379 sub lspci {
6380
6381     my $devices = {};
6382
6383     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6384             my (undef, $id, $function) = @_;
6385             my $res = { id => $id, function => $function};
6386             push @{$devices->{$id}}, $res;
6387     });
6388
6389     # Entries should be sorted by functions.
6390     foreach my $id (keys %$devices) {
6391         my $dev = $devices->{$id};
6392         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6393     }
6394
6395     return $devices;
6396 }
6397
6398 sub vm_iothreads_list {
6399     my ($vmid) = @_;
6400
6401     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6402
6403     my $iothreads = {};
6404     foreach my $iothread (@$res) {
6405         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6406     }
6407
6408     return $iothreads;
6409 }
6410
6411 sub scsihw_infos {
6412     my ($conf, $drive) = @_;
6413
6414     my $maxdev = 0;
6415
6416     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6417         $maxdev = 7;
6418     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6419         $maxdev = 1;
6420     } else {
6421         $maxdev = 256;
6422     }
6423
6424     my $controller = int($drive->{index} / $maxdev);
6425     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6426
6427     return ($maxdev, $controller, $controller_prefix);
6428 }
6429
6430 sub add_hyperv_enlightenments {
6431     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6432
6433     return if $winversion < 6;
6434     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6435
6436     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6437
6438     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6439         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6440         push @$cpuFlags , 'hv_vapic';
6441         push @$cpuFlags , 'hv_time';
6442     } else {
6443         push @$cpuFlags , 'hv_spinlocks=0xffff';
6444     }
6445
6446     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6447         push @$cpuFlags , 'hv_reset';
6448         push @$cpuFlags , 'hv_vpindex';
6449         push @$cpuFlags , 'hv_runtime';
6450     }
6451
6452     if ($winversion >= 7) {
6453         push @$cpuFlags , 'hv_relaxed';
6454     }
6455 }
6456
6457 sub windows_version {
6458     my ($ostype) = @_;
6459
6460     return 0 if !$ostype;
6461
6462     my $winversion = 0;
6463
6464     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6465         $winversion = 5;
6466     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6467         $winversion = 6;
6468     } elsif ($ostype =~ m/^win(\d+)$/) {
6469         $winversion = $1;
6470     }
6471
6472     return $winversion;
6473 }
6474
6475 sub resolve_dst_disk_format {
6476         my ($storecfg, $storeid, $src_volname, $format) = @_;
6477         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6478
6479         if (!$format) {
6480             # if no target format is specified, use the source disk format as hint
6481             if ($src_volname) {
6482                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6483                 $format = qemu_img_format($scfg, $src_volname);
6484             } else {
6485                 return $defFormat;
6486             }
6487         }
6488
6489         # test if requested format is supported - else use default
6490         my $supported = grep { $_ eq $format } @$validFormats;
6491         $format = $defFormat if !$supported;
6492         return $format;
6493 }
6494
6495 sub resolve_first_disk {
6496     my $conf = shift;
6497     my @disks = PVE::QemuServer::valid_drive_names();
6498     my $firstdisk;
6499     foreach my $ds (reverse @disks) {
6500         next if !$conf->{$ds};
6501         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6502         next if PVE::QemuServer::drive_is_cdrom($disk);
6503         $firstdisk = $ds;
6504     }
6505     return $firstdisk;
6506 }
6507
6508 sub generate_smbios1_uuid {
6509     my ($uuid, $uuid_str);
6510     UUID::generate($uuid);
6511     UUID::unparse($uuid, $uuid_str);
6512     return "uuid=$uuid_str";
6513 }
6514
6515 # bash completion helper
6516
6517 sub complete_backup_archives {
6518     my ($cmdname, $pname, $cvalue) = @_;
6519
6520     my $cfg = PVE::Storage::config();
6521
6522     my $storeid;
6523
6524     if ($cvalue =~ m/^([^:]+):/) {
6525         $storeid = $1;
6526     }
6527
6528     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6529
6530     my $res = [];
6531     foreach my $id (keys %$data) {
6532         foreach my $item (@{$data->{$id}}) {
6533             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6534             push @$res, $item->{volid} if defined($item->{volid});
6535         }
6536     }
6537
6538     return $res;
6539 }
6540
6541 my $complete_vmid_full = sub {
6542     my ($running) = @_;
6543
6544     my $idlist = vmstatus();
6545
6546     my $res = [];
6547
6548     foreach my $id (keys %$idlist) {
6549         my $d = $idlist->{$id};
6550         if (defined($running)) {
6551             next if $d->{template};
6552             next if $running && $d->{status} ne 'running';
6553             next if !$running && $d->{status} eq 'running';
6554         }
6555         push @$res, $id;
6556
6557     }
6558     return $res;
6559 };
6560
6561 sub complete_vmid {
6562     return &$complete_vmid_full();
6563 }
6564
6565 sub complete_vmid_stopped {
6566     return &$complete_vmid_full(0);
6567 }
6568
6569 sub complete_vmid_running {
6570     return &$complete_vmid_full(1);
6571 }
6572
6573 sub complete_storage {
6574
6575     my $cfg = PVE::Storage::config();
6576     my $ids = $cfg->{ids};
6577
6578     my $res = [];
6579     foreach my $sid (keys %$ids) {
6580         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6581         next if !$ids->{$sid}->{content}->{images};
6582         push @$res, $sid;
6583     }
6584
6585     return $res;
6586 }
6587
6588 sub nbd_stop {
6589     my ($vmid) = @_;
6590
6591     vm_mon_cmd($vmid, 'nbd-server-stop');
6592 }
6593
6594 1;