PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use Time::HiRes qw(gettimeofday);
  37 use File::Copy qw(copy);
  38 use URI::Escape;
  39
  40 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  41 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  42
  43 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  44
  45 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  46
  47 # Note about locking: we use flock on the config file protect
  48 # against concurent actions.
  49 # Aditionaly, we have a 'lock' setting in the config file. This
  50 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  51 # allowed when such lock is set. But you can ignore this kind of
  52 # lock with the --skiplock flag.
  53
  54 cfs_register_file('/qemu-server/',
  55                   \&parse_vm_config,
  56                   \&write_vm_config);
  57
  58 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  59     description => "Some command save/restore state from this location.",
  60     type => 'string',
  61     maxLength => 128,
  62     optional => 1,
  63 });
  64
  65 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  66     description => "The name of the snapshot.",
  67     type => 'string', format => 'pve-configid',
  68     maxLength => 40,
  69 });
  70
  71 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  72     type => 'string',
  73     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  74     description => "The drive's backing file's data format.",
  75     optional => 1,
  76 });
  77
  78 #no warnings 'redefine';
  79
  80 sub cgroups_write {
  81    my ($controller, $vmid, $option, $value) = @_;
  82
  83    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  84    PVE::ProcFSTools::write_proc_entry($path, $value);
  85
  86 }
  87
  88 my $nodename = PVE::INotify::nodename();
  89
  90 mkdir "/etc/pve/nodes/$nodename";
  91 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  92 mkdir $confdir;
  93
  94 my $var_run_tmpdir = "/var/run/qemu-server";
  95 mkdir $var_run_tmpdir;
  96
  97 my $lock_dir = "/var/lock/qemu-server";
  98 mkdir $lock_dir;
  99
 100 my $pcisysfs = "/sys/bus/pci";
 101
 102 my $cpu_vendor_list = {
 103     # Intel CPUs
 104     486 => 'GenuineIntel',
 105     pentium => 'GenuineIntel',
 106     pentium2  => 'GenuineIntel',
 107     pentium3  => 'GenuineIntel',
 108     coreduo => 'GenuineIntel',
 109     core2duo => 'GenuineIntel',
 110     Conroe  => 'GenuineIntel',
 111     Penryn  => 'GenuineIntel',
 112     Nehalem  => 'GenuineIntel',
 113     'Nehalem-IBRS'  => 'GenuineIntel',
 114     Westmere => 'GenuineIntel',
 115     'Westmere-IBRS' => 'GenuineIntel',
 116     SandyBridge => 'GenuineIntel',
 117     'SandyBridge-IBRS' => 'GenuineIntel',
 118     IvyBridge => 'GenuineIntel',
 119     'IvyBridge-IBRS' => 'GenuineIntel',
 120     Haswell => 'GenuineIntel',
 121     'Haswell-IBRS' => 'GenuineIntel',
 122     'Haswell-noTSX' => 'GenuineIntel',
 123     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 124     Broadwell => 'GenuineIntel',
 125     'Broadwell-IBRS' => 'GenuineIntel',
 126     'Broadwell-noTSX' => 'GenuineIntel',
 127     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 128     'Skylake-Client' => 'GenuineIntel',
 129     'Skylake-Client-IBRS' => 'GenuineIntel',
 130     'Skylake-Server' => 'GenuineIntel',
 131     'Skylake-Server-IBRS' => 'GenuineIntel',
 132
 133     # AMD CPUs
 134     athlon => 'AuthenticAMD',
 135     phenom  => 'AuthenticAMD',
 136     Opteron_G1  => 'AuthenticAMD',
 137     Opteron_G2  => 'AuthenticAMD',
 138     Opteron_G3  => 'AuthenticAMD',
 139     Opteron_G4  => 'AuthenticAMD',
 140     Opteron_G5  => 'AuthenticAMD',
 141     EPYC => 'AuthenticAMD',
 142     'EPYC-IBPB' => 'AuthenticAMD',
 143
 144     # generic types, use vendor from host node
 145     host => 'default',
 146     kvm32 => 'default',
 147     kvm64 => 'default',
 148     qemu32 => 'default',
 149     qemu64 => 'default',
 150     max => 'default',
 151 };
 152
 153 my $cpu_flag = qr/[+-](pcid|spec-ctrl)/;
 154
 155 my $cpu_fmt = {
 156     cputype => {
 157         description => "Emulated CPU type.",
 158         type => 'string',
 159         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 160         default => 'kvm64',
 161         default_key => 1,
 162     },
 163     hidden => {
 164         description => "Do not identify as a KVM virtual machine.",
 165         type => 'boolean',
 166         optional => 1,
 167         default => 0
 168     },
 169     flags => {
 170         description => "List of additional CPU flags separated by ';'."
 171                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 172                      . " Currently supported flags: 'pcid', 'spec-ctrl'.",
 173         format_description => '+FLAG[;-FLAG...]',
 174         type => 'string',
 175         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 176         optional => 1,
 177     },
 178 };
 179
 180 my $watchdog_fmt = {
 181     model => {
 182         default_key => 1,
 183         type => 'string',
 184         enum => [qw(i6300esb ib700)],
 185         description => "Watchdog type to emulate.",
 186         default => 'i6300esb',
 187         optional => 1,
 188     },
 189     action => {
 190         type => 'string',
 191         enum => [qw(reset shutdown poweroff pause debug none)],
 192         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 193         optional => 1,
 194     },
 195 };
 196 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 197
 198 my $confdesc = {
 199     onboot => {
 200         optional => 1,
 201         type => 'boolean',
 202         description => "Specifies whether a VM will be started during system bootup.",
 203         default => 0,
 204     },
 205     autostart => {
 206         optional => 1,
 207         type => 'boolean',
 208         description => "Automatic restart after crash (currently ignored).",
 209         default => 0,
 210     },
 211     hotplug => {
 212         optional => 1,
 213         type => 'string', format => 'pve-hotplug-features',
 214         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 215         default => 'network,disk,usb',
 216     },
 217     reboot => {
 218         optional => 1,
 219         type => 'boolean',
 220         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 221         default => 1,
 222     },
 223     lock => {
 224         optional => 1,
 225         type => 'string',
 226         description => "Lock/unlock the VM.",
 227         enum => [qw(migrate backup snapshot rollback)],
 228     },
 229     cpulimit => {
 230         optional => 1,
 231         type => 'number',
 232         description => "Limit of CPU usage.",
 233         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 234         minimum => 0,
 235         maximum => 128,
 236         default => 0,
 237     },
 238     cpuunits => {
 239         optional => 1,
 240         type => 'integer',
 241         description => "CPU weight for a VM.",
 242         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 243         minimum => 2,
 244         maximum => 262144,
 245         default => 1024,
 246     },
 247     memory => {
 248         optional => 1,
 249         type => 'integer',
 250         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 251         minimum => 16,
 252         default => 512,
 253     },
 254     balloon => {
 255         optional => 1,
 256         type => 'integer',
 257         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 258         minimum => 0,
 259     },
 260     shares => {
 261         optional => 1,
 262         type => 'integer',
 263         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 264         minimum => 0,
 265         maximum => 50000,
 266         default => 1000,
 267     },
 268     keyboard => {
 269         optional => 1,
 270         type => 'string',
 271         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.".
 272                        "It should not be necessary to set it.",
 273         enum => PVE::Tools::kvmkeymaplist(),
 274         default => undef,
 275     },
 276     name => {
 277         optional => 1,
 278         type => 'string', format => 'dns-name',
 279         description => "Set a name for the VM. Only used on the configuration web interface.",
 280     },
 281     scsihw => {
 282         optional => 1,
 283         type => 'string',
 284         description => "SCSI controller model",
 285         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 286         default => 'lsi',
 287     },
 288     description => {
 289         optional => 1,
 290         type => 'string',
 291         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 292     },
 293     ostype => {
 294         optional => 1,
 295         type => 'string',
 296         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 297         description => "Specify guest operating system.",
 298         verbose_description => <<EODESC,
 299 Specify guest operating system. This is used to enable special
 300 optimization/features for specific operating systems:
 301
 302 [horizontal]
 303 other;; unspecified OS
 304 wxp;; Microsoft Windows XP
 305 w2k;; Microsoft Windows 2000
 306 w2k3;; Microsoft Windows 2003
 307 w2k8;; Microsoft Windows 2008
 308 wvista;; Microsoft Windows Vista
 309 win7;; Microsoft Windows 7
 310 win8;; Microsoft Windows 8/2012/2012r2
 311 win10;; Microsoft Windows 10/2016
 312 l24;; Linux 2.4 Kernel
 313 l26;; Linux 2.6/3.X Kernel
 314 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 315 EODESC
 316     },
 317     boot => {
 318         optional => 1,
 319         type => 'string',
 320         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 321         pattern => '[acdn]{1,4}',
 322         default => 'cdn',
 323     },
 324     bootdisk => {
 325         optional => 1,
 326         type => 'string', format => 'pve-qm-bootdisk',
 327         description => "Enable booting from specified disk.",
 328         pattern => '(ide|sata|scsi|virtio)\d+',
 329     },
 330     smp => {
 331         optional => 1,
 332         type => 'integer',
 333         description => "The number of CPUs. Please use option -sockets instead.",
 334         minimum => 1,
 335         default => 1,
 336     },
 337     sockets => {
 338         optional => 1,
 339         type => 'integer',
 340         description => "The number of CPU sockets.",
 341         minimum => 1,
 342         default => 1,
 343     },
 344     cores => {
 345         optional => 1,
 346         type => 'integer',
 347         description => "The number of cores per socket.",
 348         minimum => 1,
 349         default => 1,
 350     },
 351     numa => {
 352         optional => 1,
 353         type => 'boolean',
 354         description => "Enable/disable NUMA.",
 355         default => 0,
 356     },
 357     hugepages => {
 358         optional => 1,
 359         type => 'string',
 360         description => "Enable/disable hugepages memory.",
 361         enum => [qw(any 2 1024)],
 362     },
 363     vcpus => {
 364         optional => 1,
 365         type => 'integer',
 366         description => "Number of hotplugged vcpus.",
 367         minimum => 1,
 368         default => 0,
 369     },
 370     acpi => {
 371         optional => 1,
 372         type => 'boolean',
 373         description => "Enable/disable ACPI.",
 374         default => 1,
 375     },
 376     agent => {
 377         optional => 1,
 378         type => 'boolean',
 379         description => "Enable/disable Qemu GuestAgent.",
 380         default => 0,
 381     },
 382     kvm => {
 383         optional => 1,
 384         type => 'boolean',
 385         description => "Enable/disable KVM hardware virtualization.",
 386         default => 1,
 387     },
 388     tdf => {
 389         optional => 1,
 390         type => 'boolean',
 391         description => "Enable/disable time drift fix.",
 392         default => 0,
 393     },
 394     localtime => {
 395         optional => 1,
 396         type => 'boolean',
 397         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 398     },
 399     freeze => {
 400         optional => 1,
 401         type => 'boolean',
 402         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 403     },
 404     vga => {
 405         optional => 1,
 406         type => 'string',
 407         description => "Select the VGA type.",
 408         verbose_description => "Select the VGA type. If you want to use high resolution" .
 409             " modes (>= 1280x1024x16) then you should use the options " .
 410             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 411             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 412             "display sever. For win* OS you can select how many independent " .
 413             "displays you want, Linux guests can add displays them self. " .
 414             "You can also run without any graphic card, using a serial device" .
 415             " as terminal.",
 416         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 417     },
 418     watchdog => {
 419         optional => 1,
 420         type => 'string', format => 'pve-qm-watchdog',
 421         description => "Create a virtual hardware watchdog device.",
 422         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 423             " (by a guest action), the watchdog must be periodically polled " .
 424             "by an agent inside the guest or else the watchdog will reset " .
 425             "the guest (or execute the respective action specified)",
 426     },
 427     startdate => {
 428         optional => 1,
 429         type => 'string',
 430         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 431         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 432         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 433         default => 'now',
 434     },
 435     startup =>  get_standard_option('pve-startup-order'),
 436     template => {
 437         optional => 1,
 438         type => 'boolean',
 439         description => "Enable/disable Template.",
 440         default => 0,
 441     },
 442     args => {
 443         optional => 1,
 444         type => 'string',
 445         description => "Arbitrary arguments passed to kvm.",
 446         verbose_description => <<EODESCR,
 447 Arbitrary arguments passed to kvm, for example:
 448
 449 args: -no-reboot -no-hpet
 450
 451 NOTE: this option is for experts only.
 452 EODESCR
 453     },
 454     tablet => {
 455         optional => 1,
 456         type => 'boolean',
 457         default => 1,
 458         description => "Enable/disable the USB tablet device.",
 459         verbose_description => "Enable/disable the USB tablet device. This device is " .
 460             "usually needed to allow absolute mouse positioning with VNC. " .
 461             "Else the mouse runs out of sync with normal VNC clients. " .
 462             "If you're running lots of console-only guests on one host, " .
 463             "you may consider disabling this to save some context switches. " .
 464             "This is turned off by default if you use spice (-vga=qxl).",
 465     },
 466     migrate_speed => {
 467         optional => 1,
 468         type => 'integer',
 469         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 470         minimum => 0,
 471         default => 0,
 472     },
 473     migrate_downtime => {
 474         optional => 1,
 475         type => 'number',
 476         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 477         minimum => 0,
 478         default => 0.1,
 479     },
 480     cdrom => {
 481         optional => 1,
 482         type => 'string', format => 'pve-qm-ide',
 483         typetext => '<volume>',
 484         description => "This is an alias for option -ide2",
 485     },
 486     cpu => {
 487         optional => 1,
 488         description => "Emulated CPU type.",
 489         type => 'string',
 490         format => $cpu_fmt,
 491     },
 492     parent => get_standard_option('pve-snapshot-name', {
 493         optional => 1,
 494         description => "Parent snapshot name. This is used internally, and should not be modified.",
 495     }),
 496     snaptime => {
 497         optional => 1,
 498         description => "Timestamp for snapshots.",
 499         type => 'integer',
 500         minimum => 0,
 501     },
 502     vmstate => {
 503         optional => 1,
 504         type => 'string', format => 'pve-volume-id',
 505         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 506     },
 507     vmstatestorage => get_standard_option('pve-storage-id', {
 508         description => "Default storage for VM state volumes/files.",
 509         optional => 1,
 510     }),
 511     machine => {
 512         description => "Specific the Qemu machine type.",
 513         type => 'string',
 514         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 515         maxLength => 40,
 516         optional => 1,
 517     },
 518     smbios1 => {
 519         description => "Specify SMBIOS type 1 fields.",
 520         type => 'string', format => 'pve-qm-smbios1',
 521         maxLength => 256,
 522         optional => 1,
 523     },
 524     protection => {
 525         optional => 1,
 526         type => 'boolean',
 527         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 528         default => 0,
 529     },
 530     bios => {
 531         optional => 1,
 532         type => 'string',
 533         enum => [ qw(seabios ovmf) ],
 534         description => "Select BIOS implementation.",
 535         default => 'seabios',
 536     },
 537 };
 538
 539 # what about other qemu settings ?
 540 #cpu => 'string',
 541 #machine => 'string',
 542 #fda => 'file',
 543 #fdb => 'file',
 544 #mtdblock => 'file',
 545 #sd => 'file',
 546 #pflash => 'file',
 547 #snapshot => 'bool',
 548 #bootp => 'file',
 549 ##tftp => 'dir',
 550 ##smb => 'dir',
 551 #kernel => 'file',
 552 #append => 'string',
 553 #initrd => 'file',
 554 ##soundhw => 'string',
 555
 556 while (my ($k, $v) = each %$confdesc) {
 557     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 558 }
 559
 560 my $MAX_IDE_DISKS = 4;
 561 my $MAX_SCSI_DISKS = 14;
 562 my $MAX_VIRTIO_DISKS = 16;
 563 my $MAX_SATA_DISKS = 6;
 564 my $MAX_USB_DEVICES = 5;
 565 my $MAX_NETS = 32;
 566 my $MAX_UNUSED_DISKS = 8;
 567 my $MAX_HOSTPCI_DEVICES = 4;
 568 my $MAX_SERIAL_PORTS = 4;
 569 my $MAX_PARALLEL_PORTS = 3;
 570 my $MAX_NUMA = 8;
 571
 572 my $numa_fmt = {
 573     cpus => {
 574         type => "string",
 575         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 576         description => "CPUs accessing this NUMA node.",
 577         format_description => "id[-id];...",
 578     },
 579     memory => {
 580         type => "number",
 581         description => "Amount of memory this NUMA node provides.",
 582         optional => 1,
 583     },
 584     hostnodes => {
 585         type => "string",
 586         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 587         description => "Host NUMA nodes to use.",
 588         format_description => "id[-id];...",
 589         optional => 1,
 590     },
 591     policy => {
 592         type => 'string',
 593         enum => [qw(preferred bind interleave)],
 594         description => "NUMA allocation policy.",
 595         optional => 1,
 596     },
 597 };
 598 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 599 my $numadesc = {
 600     optional => 1,
 601     type => 'string', format => $numa_fmt,
 602     description => "NUMA topology.",
 603 };
 604 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 605
 606 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 607     $confdesc->{"numa$i"} = $numadesc;
 608 }
 609
 610 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 611                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 612                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 613 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 614
 615 my $net_fmt_bridge_descr = <<__EOD__;
 616 Bridge to attach the network device to. The Proxmox VE standard bridge
 617 is called 'vmbr0'.
 618
 619 If you do not specify a bridge, we create a kvm user (NATed) network
 620 device, which provides DHCP and DNS services. The following addresses
 621 are used:
 622
 623  10.0.2.2   Gateway
 624  10.0.2.3   DNS Server
 625  10.0.2.4   SMB Server
 626
 627 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 628 __EOD__
 629
 630 my $net_fmt = {
 631     macaddr => {
 632         type => 'string',
 633         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 634         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 635         format_description => "XX:XX:XX:XX:XX:XX",
 636         optional => 1,
 637     },
 638     model => {
 639         type => 'string',
 640         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 641         enum => $nic_model_list,
 642         default_key => 1,
 643     },
 644     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 645     bridge => {
 646         type => 'string',
 647         description => $net_fmt_bridge_descr,
 648         format_description => 'bridge',
 649         optional => 1,
 650     },
 651     queues => {
 652         type => 'integer',
 653         minimum => 0, maximum => 16,
 654         description => 'Number of packet queues to be used on the device.',
 655         optional => 1,
 656     },
 657     rate => {
 658         type => 'number',
 659         minimum => 0,
 660         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 661         optional => 1,
 662     },
 663     tag => {
 664         type => 'integer',
 665         minimum => 1, maximum => 4094,
 666         description => 'VLAN tag to apply to packets on this interface.',
 667         optional => 1,
 668     },
 669     trunks => {
 670         type => 'string',
 671         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 672         description => 'VLAN trunks to pass through this interface.',
 673         format_description => 'vlanid[;vlanid...]',
 674         optional => 1,
 675     },
 676     firewall => {
 677         type => 'boolean',
 678         description => 'Whether this interface should be protected by the firewall.',
 679         optional => 1,
 680     },
 681     link_down => {
 682         type => 'boolean',
 683         description => 'Whether this interface should be disconnected (like pulling the plug).',
 684         optional => 1,
 685     },
 686 };
 687
 688 my $netdesc = {
 689     optional => 1,
 690     type => 'string', format => $net_fmt,
 691     description => "Specify network devices.",
 692 };
 693
 694 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 695
 696 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 697     $confdesc->{"net$i"} = $netdesc;
 698 }
 699
 700 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 701 sub verify_volume_id_or_qm_path {
 702     my ($volid, $noerr) = @_;
 703
 704     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 705         return $volid;
 706     }
 707
 708     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 709     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 710     if ($@) {
 711         return undef if $noerr;
 712         die $@;
 713     }
 714     return $volid;
 715 }
 716
 717 my $drivename_hash;
 718
 719 my %drivedesc_base = (
 720     volume => { alias => 'file' },
 721     file => {
 722         type => 'string',
 723         format => 'pve-volume-id-or-qm-path',
 724         default_key => 1,
 725         format_description => 'volume',
 726         description => "The drive's backing volume.",
 727     },
 728     media => {
 729         type => 'string',
 730         enum => [qw(cdrom disk)],
 731         description => "The drive's media type.",
 732         default => 'disk',
 733         optional => 1
 734     },
 735     cyls => {
 736         type => 'integer',
 737         description => "Force the drive's physical geometry to have a specific cylinder count.",
 738         optional => 1
 739     },
 740     heads => {
 741         type => 'integer',
 742         description => "Force the drive's physical geometry to have a specific head count.",
 743         optional => 1
 744     },
 745     secs => {
 746         type => 'integer',
 747         description => "Force the drive's physical geometry to have a specific sector count.",
 748         optional => 1
 749     },
 750     trans => {
 751         type => 'string',
 752         enum => [qw(none lba auto)],
 753         description => "Force disk geometry bios translation mode.",
 754         optional => 1,
 755     },
 756     snapshot => {
 757         type => 'boolean',
 758         description => "Controls qemu's snapshot mode feature."
 759             . " If activated, changes made to the disk are temporary and will"
 760             . " be discarded when the VM is shutdown.",
 761         optional => 1,
 762     },
 763     cache => {
 764         type => 'string',
 765         enum => [qw(none writethrough writeback unsafe directsync)],
 766         description => "The drive's cache mode",
 767         optional => 1,
 768     },
 769     format => get_standard_option('pve-qm-image-format'),
 770     size => {
 771         type => 'string',
 772         format => 'disk-size',
 773         format_description => 'DiskSize',
 774         description => "Disk size. This is purely informational and has no effect.",
 775         optional => 1,
 776     },
 777     backup => {
 778         type => 'boolean',
 779         description => "Whether the drive should be included when making backups.",
 780         optional => 1,
 781     },
 782     replicate => {
 783         type => 'boolean',
 784         description => 'Whether the drive should considered for replication jobs.',
 785         optional => 1,
 786         default => 1,
 787     },
 788     rerror => {
 789         type => 'string',
 790         enum => [qw(ignore report stop)],
 791         description => 'Read error action.',
 792         optional => 1,
 793     },
 794     werror => {
 795         type => 'string',
 796         enum => [qw(enospc ignore report stop)],
 797         description => 'Write error action.',
 798         optional => 1,
 799     },
 800     aio => {
 801         type => 'string',
 802         enum => [qw(native threads)],
 803         description => 'AIO type to use.',
 804         optional => 1,
 805     },
 806     discard => {
 807         type => 'string',
 808         enum => [qw(ignore on)],
 809         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 810         optional => 1,
 811     },
 812     detect_zeroes => {
 813         type => 'boolean',
 814         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 815         optional => 1,
 816     },
 817     serial => {
 818         type => 'string',
 819         format => 'urlencoded',
 820         format_description => 'serial',
 821         maxLength => 20*3, # *3 since it's %xx url enoded
 822         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 823         optional => 1,
 824     },
 825     shared => {
 826         type => 'boolean',
 827         description => 'Mark this locally-managed volume as available on all nodes',
 828         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 829         optional => 1,
 830         default => 0,
 831     }
 832 );
 833
 834 my %iothread_fmt = ( iothread => {
 835         type => 'boolean',
 836         description => "Whether to use iothreads for this drive",
 837         optional => 1,
 838 });
 839
 840 my %model_fmt = (
 841     model => {
 842         type => 'string',
 843         format => 'urlencoded',
 844         format_description => 'model',
 845         maxLength => 40*3, # *3 since it's %xx url enoded
 846         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 847         optional => 1,
 848     },
 849 );
 850
 851 my %queues_fmt = (
 852     queues => {
 853         type => 'integer',
 854         description => "Number of queues.",
 855         minimum => 2,
 856         optional => 1
 857     }
 858 );
 859
 860 my %scsiblock_fmt = (
 861     scsiblock => {
 862         type => 'boolean',
 863         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 864         optional => 1,
 865         default => 0,
 866     },
 867 );
 868
 869 my $add_throttle_desc = sub {
 870     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 871     my $d = {
 872         type => $type,
 873         format_description => $unit,
 874         description => "Maximum $what in $longunit.",
 875         optional => 1,
 876     };
 877     $d->{minimum} = $minimum if defined($minimum);
 878     $drivedesc_base{$key} = $d;
 879 };
 880 # throughput: (leaky bucket)
 881 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 882 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 883 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 884 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 885 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 886 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 887 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 888 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 889 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 890
 891 # pools: (pool of IO before throttling starts taking effect)
 892 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 893 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 894 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 895 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 896 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 897 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 898
 899 # burst lengths
 900 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 901 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 902 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 903 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 904 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 905 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 906
 907 # legacy support
 908 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
 909 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
 910 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
 911 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
 912
 913 my $ide_fmt = {
 914     %drivedesc_base,
 915     %model_fmt,
 916 };
 917 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
 918
 919 my $idedesc = {
 920     optional => 1,
 921     type => 'string', format => $ide_fmt,
 922     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
 923 };
 924 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
 925
 926 my $scsi_fmt = {
 927     %drivedesc_base,
 928     %iothread_fmt,
 929     %queues_fmt,
 930     %scsiblock_fmt,
 931 };
 932 my $scsidesc = {
 933     optional => 1,
 934     type => 'string', format => $scsi_fmt,
 935     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
 936 };
 937 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
 938
 939 my $sata_fmt = {
 940     %drivedesc_base,
 941 };
 942 my $satadesc = {
 943     optional => 1,
 944     type => 'string', format => $sata_fmt,
 945     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
 946 };
 947 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
 948
 949 my $virtio_fmt = {
 950     %drivedesc_base,
 951     %iothread_fmt,
 952 };
 953 my $virtiodesc = {
 954     optional => 1,
 955     type => 'string', format => $virtio_fmt,
 956     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
 957 };
 958 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
 959
 960 my $alldrive_fmt = {
 961     %drivedesc_base,
 962     %iothread_fmt,
 963     %model_fmt,
 964     %queues_fmt,
 965     %scsiblock_fmt,
 966 };
 967
 968 my $efidisk_fmt = {
 969     volume => { alias => 'file' },
 970     file => {
 971         type => 'string',
 972         format => 'pve-volume-id-or-qm-path',
 973         default_key => 1,
 974         format_description => 'volume',
 975         description => "The drive's backing volume.",
 976     },
 977     format => get_standard_option('pve-qm-image-format'),
 978     size => {
 979         type => 'string',
 980         format => 'disk-size',
 981         format_description => 'DiskSize',
 982         description => "Disk size. This is purely informational and has no effect.",
 983         optional => 1,
 984     },
 985 };
 986
 987 my $efidisk_desc = {
 988     optional => 1,
 989     type => 'string', format => $efidisk_fmt,
 990     description => "Configure a Disk for storing EFI vars",
 991 };
 992
 993 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
 994
 995 my $usb_fmt = {
 996     host => {
 997         default_key => 1,
 998         type => 'string', format => 'pve-qm-usb-device',
 999         format_description => 'HOSTUSBDEVICE|spice',
1000         description => <<EODESCR,
1001 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1002
1003  'bus-port(.port)*' (decimal numbers) or
1004  'vendor_id:product_id' (hexadeciaml numbers) or
1005  'spice'
1006
1007 You can use the 'lsusb -t' command to list existing usb devices.
1008
1009 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1010
1011 The value 'spice' can be used to add a usb redirection devices for spice.
1012 EODESCR
1013     },
1014     usb3 => {
1015         optional => 1,
1016         type => 'boolean',
1017         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1018         default => 0,
1019     },
1020 };
1021
1022 my $usbdesc = {
1023     optional => 1,
1024     type => 'string', format => $usb_fmt,
1025     description => "Configure an USB device (n is 0 to 4).",
1026 };
1027 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1028
1029 # NOTE: the match-groups of this regex are used in parse_hostpci
1030 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1031 my $hostpci_fmt = {
1032     host => {
1033         default_key => 1,
1034         type => 'string',
1035         pattern => qr/$PCIRE(;$PCIRE)*/,
1036         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1037         description => <<EODESCR,
1038 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1039 of PCI virtual functions of the host. HOSTPCIID syntax is:
1040
1041 'bus:dev.func' (hexadecimal numbers)
1042
1043 You can us the 'lspci' command to list existing PCI devices.
1044 EODESCR
1045     },
1046     rombar => {
1047         type => 'boolean',
1048         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1049         optional => 1,
1050         default => 1,
1051     },
1052     romfile => {
1053         type => 'string',
1054         pattern => '[^,;]+',
1055         format_description => 'string',
1056         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1057         optional => 1,
1058     },
1059     pcie => {
1060         type => 'boolean',
1061         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1062         optional => 1,
1063         default => 0,
1064     },
1065     'x-vga' => {
1066         type => 'boolean',
1067         description =>  "Enable vfio-vga device support.",
1068         optional => 1,
1069         default => 0,
1070     },
1071 };
1072 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1073
1074 my $hostpcidesc = {
1075         optional => 1,
1076         type => 'string', format => 'pve-qm-hostpci',
1077         description => "Map host PCI devices into guest.",
1078         verbose_description =>  <<EODESCR,
1079 Map host PCI devices into guest.
1080
1081 NOTE: This option allows direct access to host hardware. So it is no longer
1082 possible to migrate such machines - use with special care.
1083
1084 CAUTION: Experimental! User reported problems with this option.
1085 EODESCR
1086 };
1087 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1088
1089 my $serialdesc = {
1090         optional => 1,
1091         type => 'string',
1092         pattern => '(/dev/.+|socket)',
1093         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1094         verbose_description =>  <<EODESCR,
1095 Create a serial device inside the VM (n is 0 to 3), and pass through a
1096 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1097 host side (use 'qm terminal' to open a terminal connection).
1098
1099 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1100
1101 CAUTION: Experimental! User reported problems with this option.
1102 EODESCR
1103 };
1104
1105 my $paralleldesc= {
1106         optional => 1,
1107         type => 'string',
1108         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1109         description =>  "Map host parallel devices (n is 0 to 2).",
1110         verbose_description =>  <<EODESCR,
1111 Map host parallel devices (n is 0 to 2).
1112
1113 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1114
1115 CAUTION: Experimental! User reported problems with this option.
1116 EODESCR
1117 };
1118
1119 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1120     $confdesc->{"parallel$i"} = $paralleldesc;
1121 }
1122
1123 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1124     $confdesc->{"serial$i"} = $serialdesc;
1125 }
1126
1127 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1128     $confdesc->{"hostpci$i"} = $hostpcidesc;
1129 }
1130
1131 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1132     $drivename_hash->{"ide$i"} = 1;
1133     $confdesc->{"ide$i"} = $idedesc;
1134 }
1135
1136 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1137     $drivename_hash->{"sata$i"} = 1;
1138     $confdesc->{"sata$i"} = $satadesc;
1139 }
1140
1141 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1142     $drivename_hash->{"scsi$i"} = 1;
1143     $confdesc->{"scsi$i"} = $scsidesc ;
1144 }
1145
1146 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1147     $drivename_hash->{"virtio$i"} = 1;
1148     $confdesc->{"virtio$i"} = $virtiodesc;
1149 }
1150
1151 $drivename_hash->{efidisk0} = 1;
1152 $confdesc->{efidisk0} = $efidisk_desc;
1153
1154 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1155     $confdesc->{"usb$i"} = $usbdesc;
1156 }
1157
1158 my $unuseddesc = {
1159     optional => 1,
1160     type => 'string', format => 'pve-volume-id',
1161     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1162 };
1163
1164 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1165     $confdesc->{"unused$i"} = $unuseddesc;
1166 }
1167
1168 my $kvm_api_version = 0;
1169
1170 sub kvm_version {
1171
1172     return $kvm_api_version if $kvm_api_version;
1173
1174     my $fh = IO::File->new("</dev/kvm") ||
1175         return 0;
1176
1177     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1178         $kvm_api_version = $v;
1179     }
1180
1181     $fh->close();
1182
1183     return  $kvm_api_version;
1184 }
1185
1186 my $kvm_user_version;
1187
1188 sub kvm_user_version {
1189
1190     return $kvm_user_version if $kvm_user_version;
1191
1192     $kvm_user_version = 'unknown';
1193
1194     my $code = sub {
1195         my $line = shift;
1196         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1197             $kvm_user_version = $2;
1198         }
1199     };
1200
1201     eval { run_command("kvm -version", outfunc => $code); };
1202     warn $@ if $@;
1203
1204     return $kvm_user_version;
1205
1206 }
1207
1208 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1209
1210 sub valid_drive_names {
1211     # order is important - used to autoselect boot disk
1212     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1213             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1214             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1215             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1216             'efidisk0');
1217 }
1218
1219 sub is_valid_drivename {
1220     my $dev = shift;
1221
1222     return defined($drivename_hash->{$dev});
1223 }
1224
1225 sub option_exists {
1226     my $key = shift;
1227     return defined($confdesc->{$key});
1228 }
1229
1230 sub nic_models {
1231     return $nic_model_list;
1232 }
1233
1234 sub os_list_description {
1235
1236     return {
1237         other => 'Other',
1238         wxp => 'Windows XP',
1239         w2k => 'Windows 2000',
1240         w2k3 =>, 'Windows 2003',
1241         w2k8 => 'Windows 2008',
1242         wvista => 'Windows Vista',
1243         win7 => 'Windows 7',
1244         win8 => 'Windows 8/2012',
1245         win10 => 'Windows 10/2016',
1246         l24 => 'Linux 2.4',
1247         l26 => 'Linux 2.6',
1248     };
1249 }
1250
1251 my $cdrom_path;
1252
1253 sub get_cdrom_path {
1254
1255     return  $cdrom_path if $cdrom_path;
1256
1257     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1258     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1259     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1260 }
1261
1262 sub get_iso_path {
1263     my ($storecfg, $vmid, $cdrom) = @_;
1264
1265     if ($cdrom eq 'cdrom') {
1266         return get_cdrom_path();
1267     } elsif ($cdrom eq 'none') {
1268         return '';
1269     } elsif ($cdrom =~ m|^/|) {
1270         return $cdrom;
1271     } else {
1272         return PVE::Storage::path($storecfg, $cdrom);
1273     }
1274 }
1275
1276 # try to convert old style file names to volume IDs
1277 sub filename_to_volume_id {
1278     my ($vmid, $file, $media) = @_;
1279
1280     if (!($file eq 'none' || $file eq 'cdrom' ||
1281           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1282
1283         return undef if $file =~ m|/|;
1284
1285         if ($media && $media eq 'cdrom') {
1286             $file = "local:iso/$file";
1287         } else {
1288             $file = "local:$vmid/$file";
1289         }
1290     }
1291
1292     return $file;
1293 }
1294
1295 sub verify_media_type {
1296     my ($opt, $vtype, $media) = @_;
1297
1298     return if !$media;
1299
1300     my $etype;
1301     if ($media eq 'disk') {
1302         $etype = 'images';
1303     } elsif ($media eq 'cdrom') {
1304         $etype = 'iso';
1305     } else {
1306         die "internal error";
1307     }
1308
1309     return if ($vtype eq $etype);
1310
1311     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1312 }
1313
1314 sub cleanup_drive_path {
1315     my ($opt, $storecfg, $drive) = @_;
1316
1317     # try to convert filesystem paths to volume IDs
1318
1319     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1320         ($drive->{file} !~ m|^/dev/.+|) &&
1321         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1322         ($drive->{file} !~ m/^\d+$/)) {
1323         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1324         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1325         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1326         verify_media_type($opt, $vtype, $drive->{media});
1327         $drive->{file} = $volid;
1328     }
1329
1330     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1331 }
1332
1333 sub parse_hotplug_features {
1334     my ($data) = @_;
1335
1336     my $res = {};
1337
1338     return $res if $data eq '0';
1339
1340     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1341
1342     foreach my $feature (PVE::Tools::split_list($data)) {
1343         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1344             $res->{$1} = 1;
1345         } else {
1346             die "invalid hotplug feature '$feature'\n";
1347         }
1348     }
1349     return $res;
1350 }
1351
1352 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1353 sub pve_verify_hotplug_features {
1354     my ($value, $noerr) = @_;
1355
1356     return $value if parse_hotplug_features($value);
1357
1358     return undef if $noerr;
1359
1360     die "unable to parse hotplug option\n";
1361 }
1362
1363 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1364 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1365 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1366 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1367 #        [,iothread=on][,serial=serial][,model=model]
1368
1369 sub parse_drive {
1370     my ($key, $data) = @_;
1371
1372     my ($interface, $index);
1373
1374     if ($key =~ m/^([^\d]+)(\d+)$/) {
1375         $interface = $1;
1376         $index = $2;
1377     } else {
1378         return undef;
1379     }
1380
1381     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1382                                      : $confdesc->{$key}->{format};
1383     if (!$desc) {
1384         warn "invalid drive key: $key\n";
1385         return undef;
1386     }
1387     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1388     return undef if !$res;
1389     $res->{interface} = $interface;
1390     $res->{index} = $index;
1391
1392     my $error = 0;
1393     foreach my $opt (qw(bps bps_rd bps_wr)) {
1394         if (my $bps = defined(delete $res->{$opt})) {
1395             if (defined($res->{"m$opt"})) {
1396                 warn "both $opt and m$opt specified\n";
1397                 ++$error;
1398                 next;
1399             }
1400             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1401         }
1402     }
1403
1404     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1405     for my $requirement (
1406         [mbps_max => 'mbps'],
1407         [mbps_rd_max => 'mbps_rd'],
1408         [mbps_wr_max => 'mbps_wr'],
1409         [miops_max => 'miops'],
1410         [miops_rd_max => 'miops_rd'],
1411         [miops_wr_max => 'miops_wr'],
1412         [bps_max_length => 'mbps_max'],
1413         [bps_rd_max_length => 'mbps_rd_max'],
1414         [bps_wr_max_length => 'mbps_wr_max'],
1415         [iops_max_length => 'iops_max'],
1416         [iops_rd_max_length => 'iops_rd_max'],
1417         [iops_wr_max_length => 'iops_wr_max']) {
1418         my ($option, $requires) = @$requirement;
1419         if ($res->{$option} && !$res->{$requires}) {
1420             warn "$option requires $requires\n";
1421             ++$error;
1422         }
1423     }
1424
1425     return undef if $error;
1426
1427     return undef if $res->{mbps_rd} && $res->{mbps};
1428     return undef if $res->{mbps_wr} && $res->{mbps};
1429     return undef if $res->{iops_rd} && $res->{iops};
1430     return undef if $res->{iops_wr} && $res->{iops};
1431
1432     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1433         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1434         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1435         return undef if $res->{interface} eq 'virtio';
1436     }
1437
1438     if (my $size = $res->{size}) {
1439         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1440     }
1441
1442     return $res;
1443 }
1444
1445 sub print_drive {
1446     my ($vmid, $drive) = @_;
1447     my $data = { %$drive };
1448     delete $data->{$_} for qw(index interface);
1449     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1450 }
1451
1452 sub scsi_inquiry {
1453     my($fh, $noerr) = @_;
1454
1455     my $SG_IO = 0x2285;
1456     my $SG_GET_VERSION_NUM = 0x2282;
1457
1458     my $versionbuf = "\x00" x 8;
1459     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1460     if (!$ret) {
1461         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1462         return undef;
1463     }
1464     my $version = unpack("I", $versionbuf);
1465     if ($version < 30000) {
1466         die "scsi generic interface too old\n"  if !$noerr;
1467         return undef;
1468     }
1469
1470     my $buf = "\x00" x 36;
1471     my $sensebuf = "\x00" x 8;
1472     my $cmd = pack("C x3 C x1", 0x12, 36);
1473
1474     # see /usr/include/scsi/sg.h
1475     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1476
1477     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1478                       length($sensebuf), 0, length($buf), $buf,
1479                       $cmd, $sensebuf, 6000);
1480
1481     $ret = ioctl($fh, $SG_IO, $packet);
1482     if (!$ret) {
1483         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1484         return undef;
1485     }
1486
1487     my @res = unpack($sg_io_hdr_t, $packet);
1488     if ($res[17] || $res[18]) {
1489         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1490         return undef;
1491     }
1492
1493     my $res = {};
1494     (my $byte0, my $byte1, $res->{vendor},
1495      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1496
1497     $res->{removable} = $byte1 & 128 ? 1 : 0;
1498     $res->{type} = $byte0 & 31;
1499
1500     return $res;
1501 }
1502
1503 sub path_is_scsi {
1504     my ($path) = @_;
1505
1506     my $fh = IO::File->new("+<$path") || return undef;
1507     my $res = scsi_inquiry($fh, 1);
1508     close($fh);
1509
1510     return $res;
1511 }
1512
1513 sub machine_type_is_q35 {
1514     my ($conf) = @_;
1515
1516     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1517 }
1518
1519 sub print_tabletdevice_full {
1520     my ($conf) = @_;
1521
1522     my $q35 = machine_type_is_q35($conf);
1523
1524     # we use uhci for old VMs because tablet driver was buggy in older qemu
1525     my $usbbus = $q35 ? "ehci" : "uhci";
1526
1527     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1528 }
1529
1530 sub print_drivedevice_full {
1531     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1532
1533     my $device = '';
1534     my $maxdev = 0;
1535
1536     if ($drive->{interface} eq 'virtio') {
1537         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1538         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1539         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1540     } elsif ($drive->{interface} eq 'scsi') {
1541
1542         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1543         my $unit = $drive->{index} % $maxdev;
1544         my $devicetype = 'hd';
1545         my $path = '';
1546         if (drive_is_cdrom($drive)) {
1547             $devicetype = 'cd';
1548         } else {
1549             if ($drive->{file} =~ m|^/|) {
1550                 $path = $drive->{file};
1551                 if (my $info = path_is_scsi($path)) {
1552                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1553                         $devicetype = 'block';
1554                     } elsif ($info->{type} == 1) { # tape
1555                         $devicetype = 'generic';
1556                     }
1557                 }
1558             } else {
1559                  $path = PVE::Storage::path($storecfg, $drive->{file});
1560             }
1561
1562             if($path =~ m/^iscsi\:\/\//){
1563                 $devicetype = 'generic';
1564             }
1565         }
1566
1567         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1568             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1569         } else {
1570             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1571         }
1572
1573     } elsif ($drive->{interface} eq 'ide'){
1574         $maxdev = 2;
1575         my $controller = int($drive->{index} / $maxdev);
1576         my $unit = $drive->{index} % $maxdev;
1577         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1578
1579         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1580         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1581             $model = URI::Escape::uri_unescape($model);
1582             $device .= ",model=$model";
1583         }
1584     } elsif ($drive->{interface} eq 'sata'){
1585         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1586         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1587         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1588     } elsif ($drive->{interface} eq 'usb') {
1589         die "implement me";
1590         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1591     } else {
1592         die "unsupported interface type";
1593     }
1594
1595     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1596
1597     return $device;
1598 }
1599
1600 sub get_initiator_name {
1601     my $initiator;
1602
1603     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1604     while (defined(my $line = <$fh>)) {
1605         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1606         $initiator = $1;
1607         last;
1608     }
1609     $fh->close();
1610
1611     return $initiator;
1612 }
1613
1614 sub print_drive_full {
1615     my ($storecfg, $vmid, $drive) = @_;
1616
1617     my $path;
1618     my $volid = $drive->{file};
1619     my $format;
1620
1621     if (drive_is_cdrom($drive)) {
1622         $path = get_iso_path($storecfg, $vmid, $volid);
1623     } else {
1624         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1625         if ($storeid) {
1626             $path = PVE::Storage::path($storecfg, $volid);
1627             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1628             $format = qemu_img_format($scfg, $volname);
1629         } else {
1630             $path = $volid;
1631             $format = "raw";
1632         }
1633    }
1634
1635     my $opts = '';
1636     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1637     foreach my $o (@qemu_drive_options) {
1638         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1639     }
1640
1641     # snapshot only accepts on|off
1642     if (defined($drive->{snapshot})) {
1643         my $v = $drive->{snapshot} ? 'on' : 'off';
1644         $opts .= ",snapshot=$v";
1645     }
1646
1647     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1648         my ($dir, $qmpname) = @$type;
1649         if (my $v = $drive->{"mbps$dir"}) {
1650             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1651         }
1652         if (my $v = $drive->{"mbps${dir}_max"}) {
1653             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1654         }
1655         if (my $v = $drive->{"bps${dir}_max_length"}) {
1656             $opts .= ",throttling.bps$qmpname-max-length=$v";
1657         }
1658         if (my $v = $drive->{"iops${dir}"}) {
1659             $opts .= ",throttling.iops$qmpname=$v";
1660         }
1661         if (my $v = $drive->{"iops${dir}_max"}) {
1662             $opts .= ",throttling.iops$qmpname-max=$v";
1663         }
1664         if (my $v = $drive->{"iops${dir}_max_length"}) {
1665             $opts .= ",throttling.iops$qmpname-max-length=$v";
1666         }
1667     }
1668
1669     if (my $serial = $drive->{serial}) {
1670         $serial = URI::Escape::uri_unescape($serial);
1671         $opts .= ",serial=$serial";
1672     }
1673
1674     $opts .= ",format=$format" if $format && !$drive->{format};
1675
1676     my $cache_direct = 0;
1677
1678     if (my $cache = $drive->{cache}) {
1679         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1680     } elsif (!drive_is_cdrom($drive)) {
1681         $opts .= ",cache=none";
1682         $cache_direct = 1;
1683     }
1684
1685     # aio native works only with O_DIRECT
1686     if (!$drive->{aio}) {
1687         if($cache_direct) {
1688             $opts .= ",aio=native";
1689         } else {
1690             $opts .= ",aio=threads";
1691         }
1692     }
1693
1694     if (!drive_is_cdrom($drive)) {
1695         my $detectzeroes;
1696         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1697             $detectzeroes = 'off';
1698         } elsif ($drive->{discard}) {
1699             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1700         } else {
1701             # This used to be our default with discard not being specified:
1702             $detectzeroes = 'on';
1703         }
1704         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1705     }
1706
1707     my $pathinfo = $path ? "file=$path," : '';
1708
1709     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1710 }
1711
1712 sub print_netdevice_full {
1713     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1714
1715     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1716
1717     my $device = $net->{model};
1718     if ($net->{model} eq 'virtio') {
1719          $device = 'virtio-net-pci';
1720      };
1721
1722     my $pciaddr = print_pci_addr("$netid", $bridges);
1723     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1724     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1725         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1726         my $vectors = $net->{queues} * 2 + 2;
1727         $tmpstr .= ",vectors=$vectors,mq=on";
1728     }
1729     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1730
1731     if ($use_old_bios_files) {
1732         my $romfile;
1733         if ($device eq 'virtio-net-pci') {
1734             $romfile = 'pxe-virtio.rom';
1735         } elsif ($device eq 'e1000') {
1736             $romfile = 'pxe-e1000.rom';
1737         } elsif ($device eq 'ne2k') {
1738             $romfile = 'pxe-ne2k_pci.rom';
1739         } elsif ($device eq 'pcnet') {
1740             $romfile = 'pxe-pcnet.rom';
1741         } elsif ($device eq 'rtl8139') {
1742             $romfile = 'pxe-rtl8139.rom';
1743         }
1744         $tmpstr .= ",romfile=$romfile" if $romfile;
1745     }
1746
1747     return $tmpstr;
1748 }
1749
1750 sub print_netdev_full {
1751     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1752
1753     my $i = '';
1754     if ($netid =~ m/^net(\d+)$/) {
1755         $i = int($1);
1756     }
1757
1758     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1759
1760     my $ifname = "tap${vmid}i$i";
1761
1762     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1763     die "interface name '$ifname' is too long (max 15 character)\n"
1764         if length($ifname) >= 16;
1765
1766     my $vhostparam = '';
1767     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1768
1769     my $vmname = $conf->{name} || "vm$vmid";
1770
1771     my $netdev = "";
1772     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1773
1774     if ($net->{bridge}) {
1775         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1776     } else {
1777         $netdev = "type=user,id=$netid,hostname=$vmname";
1778     }
1779
1780     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1781
1782     return $netdev;
1783 }
1784
1785
1786 sub print_cpu_device {
1787     my ($conf, $id) = @_;
1788
1789     my $kvm = $conf->{kvm} // 1;
1790     my $cpu = $kvm ? "kvm64" : "qemu64";
1791     if (my $cputype = $conf->{cpu}) {
1792         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1793             or die "Cannot parse cpu description: $cputype\n";
1794         $cpu = $cpuconf->{cputype};
1795     }
1796
1797     my $cores = $conf->{cores} || 1;
1798
1799     my $current_core = ($id - 1) % $cores;
1800     my $current_socket = int(($id - 1 - $current_core)/$cores);
1801
1802     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1803 }
1804
1805 sub drive_is_cdrom {
1806     my ($drive) = @_;
1807
1808     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1809
1810 }
1811
1812 sub parse_number_sets {
1813     my ($set) = @_;
1814     my $res = [];
1815     foreach my $part (split(/;/, $set)) {
1816         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1817             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1818             push @$res, [ $1, $2 ];
1819         } else {
1820             die "invalid range: $part\n";
1821         }
1822     }
1823     return $res;
1824 }
1825
1826 sub parse_numa {
1827     my ($data) = @_;
1828
1829     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1830     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1831     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1832     return $res;
1833 }
1834
1835 sub parse_hostpci {
1836     my ($value) = @_;
1837
1838     return undef if !$value;
1839
1840     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1841
1842     my @idlist = split(/;/, $res->{host});
1843     delete $res->{host};
1844     foreach my $id (@idlist) {
1845         if ($id =~ /^$PCIRE$/) {
1846             if (defined($2)) {
1847                 push @{$res->{pciid}}, { id => $1, function => $2 };
1848             } else {
1849                 my $pcidevices = lspci($1);
1850                 $res->{pciid} = $pcidevices->{$1};
1851             }
1852         } else {
1853             # should have been caught by parse_property_string already
1854             die "failed to parse PCI id: $id\n";
1855         }
1856     }
1857     return $res;
1858 }
1859
1860 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1861 sub parse_net {
1862     my ($data) = @_;
1863
1864     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1865     if ($@) {
1866         warn $@;
1867         return undef;
1868     }
1869     if (!defined($res->{macaddr})) {
1870         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1871         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1872     }
1873     return $res;
1874 }
1875
1876 sub print_net {
1877     my $net = shift;
1878
1879     return PVE::JSONSchema::print_property_string($net, $net_fmt);
1880 }
1881
1882 sub add_random_macs {
1883     my ($settings) = @_;
1884
1885     foreach my $opt (keys %$settings) {
1886         next if $opt !~ m/^net(\d+)$/;
1887         my $net = parse_net($settings->{$opt});
1888         next if !$net;
1889         $settings->{$opt} = print_net($net);
1890     }
1891 }
1892
1893 sub vm_is_volid_owner {
1894     my ($storecfg, $vmid, $volid) = @_;
1895
1896     if ($volid !~  m|^/|) {
1897         my ($path, $owner);
1898         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
1899         if ($owner && ($owner == $vmid)) {
1900             return 1;
1901         }
1902     }
1903
1904     return undef;
1905 }
1906
1907 sub split_flagged_list {
1908     my $text = shift || '';
1909     $text =~ s/[,;]/ /g;
1910     $text =~ s/^\s+//;
1911     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
1912 }
1913
1914 sub join_flagged_list {
1915     my ($how, $lst) = @_;
1916     join $how, map { $lst->{$_} . $_ } keys %$lst;
1917 }
1918
1919 sub vmconfig_delete_pending_option {
1920     my ($conf, $key, $force) = @_;
1921
1922     delete $conf->{pending}->{$key};
1923     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1924     $pending_delete_hash->{$key} = $force ? '!' : '';
1925     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1926 }
1927
1928 sub vmconfig_undelete_pending_option {
1929     my ($conf, $key) = @_;
1930
1931     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
1932     delete $pending_delete_hash->{$key};
1933
1934     if (%$pending_delete_hash) {
1935         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1936     } else {
1937         delete $conf->{pending}->{delete};
1938     }
1939 }
1940
1941 sub vmconfig_register_unused_drive {
1942     my ($storecfg, $vmid, $conf, $drive) = @_;
1943
1944     if (!drive_is_cdrom($drive)) {
1945         my $volid = $drive->{file};
1946         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
1947             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
1948         }
1949     }
1950 }
1951
1952 sub vmconfig_cleanup_pending {
1953     my ($conf) = @_;
1954
1955     # remove pending changes when nothing changed
1956     my $changes;
1957     foreach my $opt (keys %{$conf->{pending}}) {
1958         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
1959             $changes = 1;
1960             delete $conf->{pending}->{$opt};
1961         }
1962     }
1963
1964     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
1965     my $pending_delete_hash = {};
1966     while (my ($opt, $force) = each %$current_delete_hash) {
1967         if (defined($conf->{$opt})) {
1968             $pending_delete_hash->{$opt} = $force;
1969         } else {
1970             $changes = 1;
1971         }
1972     }
1973
1974     if (%$pending_delete_hash) {
1975         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
1976     } else {
1977         delete $conf->{pending}->{delete};
1978     }
1979
1980     return $changes;
1981 }
1982
1983 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
1984 my $smbios1_fmt = {
1985     uuid => {
1986         type => 'string',
1987         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
1988         format_description => 'UUID',
1989         description => "Set SMBIOS1 UUID.",
1990         optional => 1,
1991     },
1992     version => {
1993         type => 'string',
1994         pattern => '\S+',
1995         format_description => 'string',
1996         description => "Set SMBIOS1 version.",
1997         optional => 1,
1998     },
1999     serial => {
2000         type => 'string',
2001         pattern => '\S+',
2002         format_description => 'string',
2003         description => "Set SMBIOS1 serial number.",
2004         optional => 1,
2005     },
2006     manufacturer => {
2007         type => 'string',
2008         pattern => '\S+',
2009         format_description => 'string',
2010         description => "Set SMBIOS1 manufacturer.",
2011         optional => 1,
2012     },
2013     product => {
2014         type => 'string',
2015         pattern => '\S+',
2016         format_description => 'string',
2017         description => "Set SMBIOS1 product ID.",
2018         optional => 1,
2019     },
2020     sku => {
2021         type => 'string',
2022         pattern => '\S+',
2023         format_description => 'string',
2024         description => "Set SMBIOS1 SKU string.",
2025         optional => 1,
2026     },
2027     family => {
2028         type => 'string',
2029         pattern => '\S+',
2030         format_description => 'string',
2031         description => "Set SMBIOS1 family string.",
2032         optional => 1,
2033     },
2034 };
2035
2036 sub parse_smbios1 {
2037     my ($data) = @_;
2038
2039     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2040     warn $@ if $@;
2041     return $res;
2042 }
2043
2044 sub print_smbios1 {
2045     my ($smbios1) = @_;
2046     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2047 }
2048
2049 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2050
2051 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2052 sub verify_bootdisk {
2053     my ($value, $noerr) = @_;
2054
2055     return $value if is_valid_drivename($value);
2056
2057     return undef if $noerr;
2058
2059     die "invalid boot disk '$value'\n";
2060 }
2061
2062 sub parse_watchdog {
2063     my ($value) = @_;
2064
2065     return undef if !$value;
2066
2067     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2068     warn $@ if $@;
2069     return $res;
2070 }
2071
2072 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2073 sub verify_usb_device {
2074     my ($value, $noerr) = @_;
2075
2076     return $value if parse_usb_device($value);
2077
2078     return undef if $noerr;
2079
2080     die "unable to parse usb device\n";
2081 }
2082
2083 # add JSON properties for create and set function
2084 sub json_config_properties {
2085     my $prop = shift;
2086
2087     foreach my $opt (keys %$confdesc) {
2088         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2089         $prop->{$opt} = $confdesc->{$opt};
2090     }
2091
2092     return $prop;
2093 }
2094
2095 sub check_type {
2096     my ($key, $value) = @_;
2097
2098     die "unknown setting '$key'\n" if !$confdesc->{$key};
2099
2100     my $type = $confdesc->{$key}->{type};
2101
2102     if (!defined($value)) {
2103         die "got undefined value\n";
2104     }
2105
2106     if ($value =~ m/[\n\r]/) {
2107         die "property contains a line feed\n";
2108     }
2109
2110     if ($type eq 'boolean') {
2111         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2112         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2113         die "type check ('boolean') failed - got '$value'\n";
2114     } elsif ($type eq 'integer') {
2115         return int($1) if $value =~ m/^(\d+)$/;
2116         die "type check ('integer') failed - got '$value'\n";
2117     } elsif ($type eq 'number') {
2118         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2119         die "type check ('number') failed - got '$value'\n";
2120     } elsif ($type eq 'string') {
2121         if (my $fmt = $confdesc->{$key}->{format}) {
2122             PVE::JSONSchema::check_format($fmt, $value);
2123             return $value;
2124         }
2125         $value =~ s/^\"(.*)\"$/$1/;
2126         return $value;
2127     } else {
2128         die "internal error"
2129     }
2130 }
2131
2132 sub check_iommu_support{
2133     #fixme : need to check IOMMU support
2134     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2135
2136     my $iommu=1;
2137     return $iommu;
2138
2139 }
2140
2141 sub touch_config {
2142     my ($vmid) = @_;
2143
2144     my $conf = PVE::QemuConfig->config_file($vmid);
2145     utime undef, undef, $conf;
2146 }
2147
2148 sub destroy_vm {
2149     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2150
2151     my $conffile = PVE::QemuConfig->config_file($vmid);
2152
2153     my $conf = PVE::QemuConfig->load_config($vmid);
2154
2155     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2156
2157     if ($conf->{template}) {
2158         # check if any base image is still used by a linked clone
2159         foreach_drive($conf, sub {
2160                 my ($ds, $drive) = @_;
2161
2162                 return if drive_is_cdrom($drive);
2163
2164                 my $volid = $drive->{file};
2165
2166                 return if !$volid || $volid =~ m|^/|;
2167
2168                 die "base volume '$volid' is still in use by linked cloned\n"
2169                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2170
2171         });
2172     }
2173
2174     # only remove disks owned by this VM
2175     foreach_drive($conf, sub {
2176         my ($ds, $drive) = @_;
2177
2178         return if drive_is_cdrom($drive);
2179
2180         my $volid = $drive->{file};
2181
2182         return if !$volid || $volid =~ m|^/|;
2183
2184         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2185         return if !$path || !$owner || ($owner != $vmid);
2186
2187         eval {
2188             PVE::Storage::vdisk_free($storecfg, $volid);
2189         };
2190         warn "Could not remove disk '$volid', check manually: $@" if $@;
2191
2192     });
2193
2194     if ($keep_empty_config) {
2195         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2196     } else {
2197         unlink $conffile;
2198     }
2199
2200     # also remove unused disk
2201     eval {
2202         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2203
2204         eval {
2205             PVE::Storage::foreach_volid($dl, sub {
2206                 my ($volid, $sid, $volname, $d) = @_;
2207                 PVE::Storage::vdisk_free($storecfg, $volid);
2208             });
2209         };
2210         warn $@ if $@;
2211
2212     };
2213     warn $@ if $@;
2214 }
2215
2216 sub parse_vm_config {
2217     my ($filename, $raw) = @_;
2218
2219     return undef if !defined($raw);
2220
2221     my $res = {
2222         digest => Digest::SHA::sha1_hex($raw),
2223         snapshots => {},
2224         pending => {},
2225     };
2226
2227     $filename =~ m|/qemu-server/(\d+)\.conf$|
2228         || die "got strange filename '$filename'";
2229
2230     my $vmid = $1;
2231
2232     my $conf = $res;
2233     my $descr;
2234     my $section = '';
2235
2236     my @lines = split(/\n/, $raw);
2237     foreach my $line (@lines) {
2238         next if $line =~ m/^\s*$/;
2239
2240         if ($line =~ m/^\[PENDING\]\s*$/i) {
2241             $section = 'pending';
2242             if (defined($descr)) {
2243                 $descr =~ s/\s+$//;
2244                 $conf->{description} = $descr;
2245             }
2246             $descr = undef;
2247             $conf = $res->{$section} = {};
2248             next;
2249
2250         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2251             $section = $1;
2252             if (defined($descr)) {
2253                 $descr =~ s/\s+$//;
2254                 $conf->{description} = $descr;
2255             }
2256             $descr = undef;
2257             $conf = $res->{snapshots}->{$section} = {};
2258             next;
2259         }
2260
2261         if ($line =~ m/^\#(.*)\s*$/) {
2262             $descr = '' if !defined($descr);
2263             $descr .= PVE::Tools::decode_text($1) . "\n";
2264             next;
2265         }
2266
2267         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2268             $descr = '' if !defined($descr);
2269             $descr .= PVE::Tools::decode_text($2);
2270         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2271             $conf->{snapstate} = $1;
2272         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2273             my $key = $1;
2274             my $value = $2;
2275             $conf->{$key} = $value;
2276         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2277             my $value = $1;
2278             if ($section eq 'pending') {
2279                 $conf->{delete} = $value; # we parse this later
2280             } else {
2281                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2282             }
2283         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(\S+)\s*$/) {
2284             my $key = $1;
2285             my $value = $2;
2286             eval { $value = check_type($key, $value); };
2287             if ($@) {
2288                 warn "vm $vmid - unable to parse value of '$key' - $@";
2289             } else {
2290                 $key = 'ide2' if $key eq 'cdrom';
2291                 my $fmt = $confdesc->{$key}->{format};
2292                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2293                     my $v = parse_drive($key, $value);
2294                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2295                         $v->{file} = $volid;
2296                         $value = print_drive($vmid, $v);
2297                     } else {
2298                         warn "vm $vmid - unable to parse value of '$key'\n";
2299                         next;
2300                     }
2301                 }
2302
2303                 $conf->{$key} = $value;
2304             }
2305         }
2306     }
2307
2308     if (defined($descr)) {
2309         $descr =~ s/\s+$//;
2310         $conf->{description} = $descr;
2311     }
2312     delete $res->{snapstate}; # just to be sure
2313
2314     return $res;
2315 }
2316
2317 sub write_vm_config {
2318     my ($filename, $conf) = @_;
2319
2320     delete $conf->{snapstate}; # just to be sure
2321
2322     if ($conf->{cdrom}) {
2323         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2324         $conf->{ide2} = $conf->{cdrom};
2325         delete $conf->{cdrom};
2326     }
2327
2328     # we do not use 'smp' any longer
2329     if ($conf->{sockets}) {
2330         delete $conf->{smp};
2331     } elsif ($conf->{smp}) {
2332         $conf->{sockets} = $conf->{smp};
2333         delete $conf->{cores};
2334         delete $conf->{smp};
2335     }
2336
2337     my $used_volids = {};
2338
2339     my $cleanup_config = sub {
2340         my ($cref, $pending, $snapname) = @_;
2341
2342         foreach my $key (keys %$cref) {
2343             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2344                 $key eq 'snapstate' || $key eq 'pending';
2345             my $value = $cref->{$key};
2346             if ($key eq 'delete') {
2347                 die "propertry 'delete' is only allowed in [PENDING]\n"
2348                     if !$pending;
2349                 # fixme: check syntax?
2350                 next;
2351             }
2352             eval { $value = check_type($key, $value); };
2353             die "unable to parse value of '$key' - $@" if $@;
2354
2355             $cref->{$key} = $value;
2356
2357             if (!$snapname && is_valid_drivename($key)) {
2358                 my $drive = parse_drive($key, $value);
2359                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2360             }
2361         }
2362     };
2363
2364     &$cleanup_config($conf);
2365
2366     &$cleanup_config($conf->{pending}, 1);
2367
2368     foreach my $snapname (keys %{$conf->{snapshots}}) {
2369         die "internal error" if $snapname eq 'pending';
2370         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2371     }
2372
2373     # remove 'unusedX' settings if we re-add a volume
2374     foreach my $key (keys %$conf) {
2375         my $value = $conf->{$key};
2376         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2377             delete $conf->{$key};
2378         }
2379     }
2380
2381     my $generate_raw_config = sub {
2382         my ($conf, $pending) = @_;
2383
2384         my $raw = '';
2385
2386         # add description as comment to top of file
2387         if (defined(my $descr = $conf->{description})) {
2388             if ($descr) {
2389                 foreach my $cl (split(/\n/, $descr)) {
2390                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2391                 }
2392             } else {
2393                 $raw .= "#\n" if $pending;
2394             }
2395         }
2396
2397         foreach my $key (sort keys %$conf) {
2398             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2399             $raw .= "$key: $conf->{$key}\n";
2400         }
2401         return $raw;
2402     };
2403
2404     my $raw = &$generate_raw_config($conf);
2405
2406     if (scalar(keys %{$conf->{pending}})){
2407         $raw .= "\n[PENDING]\n";
2408         $raw .= &$generate_raw_config($conf->{pending}, 1);
2409     }
2410
2411     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2412         $raw .= "\n[$snapname]\n";
2413         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2414     }
2415
2416     return $raw;
2417 }
2418
2419 sub load_defaults {
2420
2421     my $res = {};
2422
2423     # we use static defaults from our JSON schema configuration
2424     foreach my $key (keys %$confdesc) {
2425         if (defined(my $default = $confdesc->{$key}->{default})) {
2426             $res->{$key} = $default;
2427         }
2428     }
2429
2430     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2431     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2432
2433     return $res;
2434 }
2435
2436 sub config_list {
2437     my $vmlist = PVE::Cluster::get_vmlist();
2438     my $res = {};
2439     return $res if !$vmlist || !$vmlist->{ids};
2440     my $ids = $vmlist->{ids};
2441
2442     foreach my $vmid (keys %$ids) {
2443         my $d = $ids->{$vmid};
2444         next if !$d->{node} || $d->{node} ne $nodename;
2445         next if !$d->{type} || $d->{type} ne 'qemu';
2446         $res->{$vmid}->{exists} = 1;
2447     }
2448     return $res;
2449 }
2450
2451 # test if VM uses local resources (to prevent migration)
2452 sub check_local_resources {
2453     my ($conf, $noerr) = @_;
2454
2455     my $loc_res = 0;
2456
2457     $loc_res = 1 if $conf->{hostusb}; # old syntax
2458     $loc_res = 1 if $conf->{hostpci}; # old syntax
2459
2460     foreach my $k (keys %$conf) {
2461         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2462         # sockets are safe: they will recreated be on the target side post-migrate
2463         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2464         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2465     }
2466
2467     die "VM uses local resources\n" if $loc_res && !$noerr;
2468
2469     return $loc_res;
2470 }
2471
2472 # check if used storages are available on all nodes (use by migrate)
2473 sub check_storage_availability {
2474     my ($storecfg, $conf, $node) = @_;
2475
2476     foreach_drive($conf, sub {
2477         my ($ds, $drive) = @_;
2478
2479         my $volid = $drive->{file};
2480         return if !$volid;
2481
2482         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2483         return if !$sid;
2484
2485         # check if storage is available on both nodes
2486         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2487         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2488    });
2489 }
2490
2491 # list nodes where all VM images are available (used by has_feature API)
2492 sub shared_nodes {
2493     my ($conf, $storecfg) = @_;
2494
2495     my $nodelist = PVE::Cluster::get_nodelist();
2496     my $nodehash = { map { $_ => 1 } @$nodelist };
2497     my $nodename = PVE::INotify::nodename();
2498
2499     foreach_drive($conf, sub {
2500         my ($ds, $drive) = @_;
2501
2502         my $volid = $drive->{file};
2503         return if !$volid;
2504
2505         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2506         if ($storeid) {
2507             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2508             if ($scfg->{disable}) {
2509                 $nodehash = {};
2510             } elsif (my $avail = $scfg->{nodes}) {
2511                 foreach my $node (keys %$nodehash) {
2512                     delete $nodehash->{$node} if !$avail->{$node};
2513                 }
2514             } elsif (!$scfg->{shared}) {
2515                 foreach my $node (keys %$nodehash) {
2516                     delete $nodehash->{$node} if $node ne $nodename
2517                 }
2518             }
2519         }
2520     });
2521
2522     return $nodehash
2523 }
2524
2525 sub check_cmdline {
2526     my ($pidfile, $pid) = @_;
2527
2528     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2529     if (defined($fh)) {
2530         my $line = <$fh>;
2531         $fh->close;
2532         return undef if !$line;
2533         my @param = split(/\0/, $line);
2534
2535         my $cmd = $param[0];
2536         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2537
2538         for (my $i = 0; $i < scalar (@param); $i++) {
2539             my $p = $param[$i];
2540             next if !$p;
2541             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2542                 my $p = $param[$i+1];
2543                 return 1 if $p && ($p eq $pidfile);
2544                 return undef;
2545             }
2546         }
2547     }
2548     return undef;
2549 }
2550
2551 sub check_running {
2552     my ($vmid, $nocheck, $node) = @_;
2553
2554     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2555
2556     die "unable to find configuration file for VM $vmid - no such machine\n"
2557         if !$nocheck && ! -f $filename;
2558
2559     my $pidfile = pidfile_name($vmid);
2560
2561     if (my $fd = IO::File->new("<$pidfile")) {
2562         my $st = stat($fd);
2563         my $line = <$fd>;
2564         close($fd);
2565
2566         my $mtime = $st->mtime;
2567         if ($mtime > time()) {
2568             warn "file '$filename' modified in future\n";
2569         }
2570
2571         if ($line =~ m/^(\d+)$/) {
2572             my $pid = $1;
2573             if (check_cmdline($pidfile, $pid)) {
2574                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2575                     return $pid;
2576                 }
2577             }
2578         }
2579     }
2580
2581     return undef;
2582 }
2583
2584 sub vzlist {
2585
2586     my $vzlist = config_list();
2587
2588     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2589
2590     while (defined(my $de = $fd->read)) {
2591         next if $de !~ m/^(\d+)\.pid$/;
2592         my $vmid = $1;
2593         next if !defined($vzlist->{$vmid});
2594         if (my $pid = check_running($vmid)) {
2595             $vzlist->{$vmid}->{pid} = $pid;
2596         }
2597     }
2598
2599     return $vzlist;
2600 }
2601
2602 sub disksize {
2603     my ($storecfg, $conf) = @_;
2604
2605     my $bootdisk = $conf->{bootdisk};
2606     return undef if !$bootdisk;
2607     return undef if !is_valid_drivename($bootdisk);
2608
2609     return undef if !$conf->{$bootdisk};
2610
2611     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2612     return undef if !defined($drive);
2613
2614     return undef if drive_is_cdrom($drive);
2615
2616     my $volid = $drive->{file};
2617     return undef if !$volid;
2618
2619     return $drive->{size};
2620 }
2621
2622 my $last_proc_pid_stat;
2623
2624 # get VM status information
2625 # This must be fast and should not block ($full == false)
2626 # We only query KVM using QMP if $full == true (this can be slow)
2627 sub vmstatus {
2628     my ($opt_vmid, $full) = @_;
2629
2630     my $res = {};
2631
2632     my $storecfg = PVE::Storage::config();
2633
2634     my $list = vzlist();
2635     my $defaults = load_defaults();
2636
2637     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2638
2639     my $cpucount = $cpuinfo->{cpus} || 1;
2640
2641     foreach my $vmid (keys %$list) {
2642         next if $opt_vmid && ($vmid ne $opt_vmid);
2643
2644         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2645         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2646
2647         my $d = {};
2648         $d->{pid} = $list->{$vmid}->{pid};
2649
2650         # fixme: better status?
2651         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2652
2653         my $size = disksize($storecfg, $conf);
2654         if (defined($size)) {
2655             $d->{disk} = 0; # no info available
2656             $d->{maxdisk} = $size;
2657         } else {
2658             $d->{disk} = 0;
2659             $d->{maxdisk} = 0;
2660         }
2661
2662         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2663             * ($conf->{cores} || $defaults->{cores});
2664         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2665         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2666
2667         $d->{name} = $conf->{name} || "VM $vmid";
2668         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2669             : $defaults->{memory}*(1024*1024);
2670
2671         if ($conf->{balloon}) {
2672             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2673             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2674                 : $defaults->{shares};
2675         }
2676
2677         $d->{uptime} = 0;
2678         $d->{cpu} = 0;
2679         $d->{mem} = 0;
2680
2681         $d->{netout} = 0;
2682         $d->{netin} = 0;
2683
2684         $d->{diskread} = 0;
2685         $d->{diskwrite} = 0;
2686
2687         $d->{template} = PVE::QemuConfig->is_template($conf);
2688
2689         $d->{serial} = 1 if conf_has_serial($conf);
2690
2691         $res->{$vmid} = $d;
2692     }
2693
2694     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2695     foreach my $dev (keys %$netdev) {
2696         next if $dev !~ m/^tap([1-9]\d*)i/;
2697         my $vmid = $1;
2698         my $d = $res->{$vmid};
2699         next if !$d;
2700
2701         $d->{netout} += $netdev->{$dev}->{receive};
2702         $d->{netin} += $netdev->{$dev}->{transmit};
2703
2704         if ($full) {
2705             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2706             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2707         }
2708
2709     }
2710
2711     my $ctime = gettimeofday;
2712
2713     foreach my $vmid (keys %$list) {
2714
2715         my $d = $res->{$vmid};
2716         my $pid = $d->{pid};
2717         next if !$pid;
2718
2719         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2720         next if !$pstat; # not running
2721
2722         my $used = $pstat->{utime} + $pstat->{stime};
2723
2724         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2725
2726         if ($pstat->{vsize}) {
2727             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2728         }
2729
2730         my $old = $last_proc_pid_stat->{$pid};
2731         if (!$old) {
2732             $last_proc_pid_stat->{$pid} = {
2733                 time => $ctime,
2734                 used => $used,
2735                 cpu => 0,
2736             };
2737             next;
2738         }
2739
2740         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2741
2742         if ($dtime > 1000) {
2743             my $dutime = $used -  $old->{used};
2744
2745             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2746             $last_proc_pid_stat->{$pid} = {
2747                 time => $ctime,
2748                 used => $used,
2749                 cpu => $d->{cpu},
2750             };
2751         } else {
2752             $d->{cpu} = $old->{cpu};
2753         }
2754     }
2755
2756     return $res if !$full;
2757
2758     my $qmpclient = PVE::QMPClient->new();
2759
2760     my $ballooncb = sub {
2761         my ($vmid, $resp) = @_;
2762
2763         my $info = $resp->{'return'};
2764         return if !$info->{max_mem};
2765
2766         my $d = $res->{$vmid};
2767
2768         # use memory assigned to VM
2769         $d->{maxmem} = $info->{max_mem};
2770         $d->{balloon} = $info->{actual};
2771
2772         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2773             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2774             $d->{freemem} = $info->{free_mem};
2775         }
2776
2777         $d->{ballooninfo} = $info;
2778     };
2779
2780     my $blockstatscb = sub {
2781         my ($vmid, $resp) = @_;
2782         my $data = $resp->{'return'} || [];
2783         my $totalrdbytes = 0;
2784         my $totalwrbytes = 0;
2785
2786         for my $blockstat (@$data) {
2787             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2788             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2789
2790             $blockstat->{device} =~ s/drive-//;
2791             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2792         }
2793         $res->{$vmid}->{diskread} = $totalrdbytes;
2794         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2795     };
2796
2797     my $statuscb = sub {
2798         my ($vmid, $resp) = @_;
2799
2800         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2801         # this fails if ballon driver is not loaded, so this must be
2802         # the last commnand (following command are aborted if this fails).
2803         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2804
2805         my $status = 'unknown';
2806         if (!defined($status = $resp->{'return'}->{status})) {
2807             warn "unable to get VM status\n";
2808             return;
2809         }
2810
2811         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2812     };
2813
2814     foreach my $vmid (keys %$list) {
2815         next if $opt_vmid && ($vmid ne $opt_vmid);
2816         next if !$res->{$vmid}->{pid}; # not running
2817         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2818     }
2819
2820     $qmpclient->queue_execute(undef, 2);
2821
2822     foreach my $vmid (keys %$list) {
2823         next if $opt_vmid && ($vmid ne $opt_vmid);
2824         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2825     }
2826
2827     return $res;
2828 }
2829
2830 sub foreach_drive {
2831     my ($conf, $func, @param) = @_;
2832
2833     foreach my $ds (valid_drive_names()) {
2834         next if !defined($conf->{$ds});
2835
2836         my $drive = parse_drive($ds, $conf->{$ds});
2837         next if !$drive;
2838
2839         &$func($ds, $drive, @param);
2840     }
2841 }
2842
2843 sub foreach_volid {
2844     my ($conf, $func, @param) = @_;
2845
2846     my $volhash = {};
2847
2848     my $test_volid = sub {
2849         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
2850
2851         return if !$volid;
2852
2853         $volhash->{$volid}->{cdrom} //= 1;
2854         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2855
2856         $volhash->{$volid}->{replicate} //= 0;
2857         $volhash->{$volid}->{replicate} = 1 if $replicate;
2858
2859         $volhash->{$volid}->{shared} //= 0;
2860         $volhash->{$volid}->{shared} = 1 if $shared;
2861
2862         $volhash->{$volid}->{referenced_in_config} //= 0;
2863         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
2864
2865         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
2866             if defined($snapname);
2867     };
2868
2869     foreach_drive($conf, sub {
2870         my ($ds, $drive) = @_;
2871         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
2872     });
2873
2874     foreach my $snapname (keys %{$conf->{snapshots}}) {
2875         my $snap = $conf->{snapshots}->{$snapname};
2876         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
2877         foreach_drive($snap, sub {
2878             my ($ds, $drive) = @_;
2879             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
2880         });
2881     }
2882
2883     foreach my $volid (keys %$volhash) {
2884         &$func($volid, $volhash->{$volid}, @param);
2885     }
2886 }
2887
2888 sub conf_has_serial {
2889     my ($conf) = @_;
2890
2891     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
2892         if ($conf->{"serial$i"}) {
2893             return 1;
2894         }
2895     }
2896
2897     return 0;
2898 }
2899
2900 sub vga_conf_has_spice {
2901     my ($vga) = @_;
2902
2903     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
2904
2905     return $1 || 1;
2906 }
2907
2908 sub config_to_command {
2909     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
2910
2911     my $cmd = [];
2912     my $globalFlags = [];
2913     my $machineFlags = [];
2914     my $rtcFlags = [];
2915     my $cpuFlags = [];
2916     my $devices = [];
2917     my $pciaddr = '';
2918     my $bridges = {};
2919     my $kvmver = kvm_user_version();
2920     my $vernum = 0; # unknown
2921     my $ostype = $conf->{ostype};
2922     my $winversion = windows_version($ostype);
2923     my $kvm = $conf->{kvm} // 1;
2924
2925     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
2926
2927     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
2928         $vernum = $1*1000000+$2*1000;
2929     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
2930         $vernum = $1*1000000+$2*1000+$3;
2931     }
2932
2933     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
2934
2935     my $have_ovz = -f '/proc/vz/vestat';
2936
2937     my $q35 = machine_type_is_q35($conf);
2938     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
2939     my $machine_type = $forcemachine || $conf->{machine};
2940     my $use_old_bios_files = undef;
2941     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
2942
2943     my $cpuunits = defined($conf->{cpuunits}) ?
2944             $conf->{cpuunits} : $defaults->{cpuunits};
2945
2946     push @$cmd, '/usr/bin/kvm';
2947
2948     push @$cmd, '-id', $vmid;
2949
2950     my $use_virtio = 0;
2951
2952     my $qmpsocket = qmp_socket($vmid);
2953     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
2954     push @$cmd, '-mon', "chardev=qmp,mode=control";
2955
2956
2957     push @$cmd, '-pidfile' , pidfile_name($vmid);
2958
2959     push @$cmd, '-daemonize';
2960
2961     if ($conf->{smbios1}) {
2962         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
2963     }
2964
2965     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
2966         die "uefi base image not found\n" if ! -f $OVMF_CODE;
2967
2968         my $path;
2969         my $format;
2970         if (my $efidisk = $conf->{efidisk0}) {
2971             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
2972             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
2973             $format = $d->{format};
2974             if ($storeid) {
2975                 $path = PVE::Storage::path($storecfg, $d->{file});
2976                 if (!defined($format)) {
2977                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2978                     $format = qemu_img_format($scfg, $volname);
2979                 }
2980             } else {
2981                 $path = $d->{file};
2982                 die "efidisk format must be specified\n"
2983                     if !defined($format);
2984             }
2985         } else {
2986             warn "no efidisk configured! Using temporary efivars disk.\n";
2987             $path = "/tmp/$vmid-ovmf.fd";
2988             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
2989             $format = 'raw';
2990         }
2991
2992         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
2993         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
2994     }
2995
2996
2997     # add usb controllers
2998     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
2999     push @$devices, @usbcontrollers if @usbcontrollers;
3000     my $vga = $conf->{vga};
3001
3002     my $qxlnum = vga_conf_has_spice($vga);
3003     $vga = 'qxl' if $qxlnum;
3004
3005     if (!$vga) {
3006         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3007             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3008         } else {
3009             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
3010         }
3011     }
3012
3013     # enable absolute mouse coordinates (needed by vnc)
3014     my $tablet;
3015     if (defined($conf->{tablet})) {
3016         $tablet = $conf->{tablet};
3017     } else {
3018         $tablet = $defaults->{tablet};
3019         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3020         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3021     }
3022
3023     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3024
3025     my $kvm_off = 0;
3026     my $gpu_passthrough;
3027
3028     # host pci devices
3029     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3030         my $d = parse_hostpci($conf->{"hostpci$i"});
3031         next if !$d;
3032
3033         my $pcie = $d->{pcie};
3034         if($pcie){
3035             die "q35 machine model is not enabled" if !$q35;
3036             $pciaddr = print_pcie_addr("hostpci$i");
3037         }else{
3038             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3039         }
3040
3041         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3042         my $romfile = $d->{romfile};
3043
3044         my $xvga = '';
3045         if ($d->{'x-vga'}) {
3046             $xvga = ',x-vga=on';
3047             $kvm_off = 1;
3048             $vga = 'none';
3049             $gpu_passthrough = 1;
3050
3051             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3052                 $xvga = "";
3053             }
3054         }
3055         my $pcidevices = $d->{pciid};
3056         my $multifunction = 1 if @$pcidevices > 1;
3057
3058         my $j=0;
3059         foreach my $pcidevice (@$pcidevices) {
3060
3061             my $id = "hostpci$i";
3062             $id .= ".$j" if $multifunction;
3063             my $addr = $pciaddr;
3064             $addr .= ".$j" if $multifunction;
3065             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3066
3067             if($j == 0){
3068                 $devicestr .= "$rombar$xvga";
3069                 $devicestr .= ",multifunction=on" if $multifunction;
3070                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3071             }
3072
3073             push @$devices, '-device', $devicestr;
3074             $j++;
3075         }
3076     }
3077
3078     # usb devices
3079     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3080     push @$devices, @usbdevices if @usbdevices;
3081     # serial devices
3082     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3083         if (my $path = $conf->{"serial$i"}) {
3084             if ($path eq 'socket') {
3085                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3086                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3087                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3088             } else {
3089                 die "no such serial device\n" if ! -c $path;
3090                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3091                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3092             }
3093         }
3094     }
3095
3096     # parallel devices
3097     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3098         if (my $path = $conf->{"parallel$i"}) {
3099             die "no such parallel device\n" if ! -c $path;
3100             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3101             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3102             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3103         }
3104     }
3105
3106     my $vmname = $conf->{name} || "vm$vmid";
3107
3108     push @$cmd, '-name', $vmname;
3109
3110     my $sockets = 1;
3111     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3112     $sockets = $conf->{sockets} if  $conf->{sockets};
3113
3114     my $cores = $conf->{cores} || 1;
3115
3116     my $maxcpus = $sockets * $cores;
3117
3118     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3119
3120     my $allowed_vcpus = $cpuinfo->{cpus};
3121
3122     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3123         if ($allowed_vcpus < $maxcpus);
3124
3125     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3126
3127         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3128         for (my $i = 2; $i <= $vcpus; $i++)  {
3129             my $cpustr = print_cpu_device($conf,$i);
3130             push @$cmd, '-device', $cpustr;
3131         }
3132
3133     } else {
3134
3135         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3136     }
3137     push @$cmd, '-nodefaults';
3138
3139     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3140
3141     my $bootindex_hash = {};
3142     my $i = 1;
3143     foreach my $o (split(//, $bootorder)) {
3144         $bootindex_hash->{$o} = $i*100;
3145         $i++;
3146     }
3147
3148     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3149
3150     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3151
3152     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3153
3154     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3155
3156     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3157         my $socket = vnc_socket($vmid);
3158         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3159     } else {
3160         push @$cmd, '-nographic';
3161     }
3162
3163     # time drift fix
3164     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3165
3166     my $useLocaltime = $conf->{localtime};
3167
3168     if ($winversion >= 5) { # windows
3169         $useLocaltime = 1 if !defined($conf->{localtime});
3170
3171         # use time drift fix when acpi is enabled
3172         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3173             $tdf = 1 if !defined($conf->{tdf});
3174         }
3175     }
3176
3177     if ($winversion >= 6) {
3178         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3179         push @$cmd, '-no-hpet';
3180     }
3181
3182     push @$rtcFlags, 'driftfix=slew' if $tdf;
3183
3184     if (!$kvm) {
3185         push @$machineFlags, 'accel=tcg';
3186     }
3187
3188     if ($machine_type) {
3189         push @$machineFlags, "type=${machine_type}";
3190     }
3191
3192     if ($conf->{startdate}) {
3193         push @$rtcFlags, "base=$conf->{startdate}";
3194     } elsif ($useLocaltime) {
3195         push @$rtcFlags, 'base=localtime';
3196     }
3197
3198     my $cpu = $kvm ? "kvm64" : "qemu64";
3199     if (my $cputype = $conf->{cpu}) {
3200         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3201             or die "Cannot parse cpu description: $cputype\n";
3202         $cpu = $cpuconf->{cputype};
3203         $kvm_off = 1 if $cpuconf->{hidden};
3204
3205         if (defined(my $flags = $cpuconf->{flags})) {
3206             push @$cpuFlags, split(";", $flags);
3207         }
3208     }
3209
3210     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3211
3212     push @$cpuFlags , '-x2apic'
3213         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3214
3215     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3216
3217     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3218
3219     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3220
3221         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3222         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3223     }
3224
3225     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3226
3227     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3228
3229     push @$cpuFlags, 'kvm=off' if $kvm_off;
3230
3231     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3232         die "internal error"; # should not happen
3233
3234     push @$cpuFlags, "vendor=${cpu_vendor}"
3235         if $cpu_vendor ne 'default';
3236
3237     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3238
3239     push @$cmd, '-cpu', $cpu;
3240
3241     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3242
3243     push @$cmd, '-S' if $conf->{freeze};
3244
3245     # set keyboard layout
3246     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3247     push @$cmd, '-k', $kb if $kb;
3248
3249     # enable sound
3250     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3251     #push @$cmd, '-soundhw', 'es1370';
3252     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3253
3254     if($conf->{agent}) {
3255         my $qgasocket = qmp_socket($vmid, 1);
3256         my $pciaddr = print_pci_addr("qga0", $bridges);
3257         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3258         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3259         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3260     }
3261
3262     my $spice_port;
3263
3264     if ($qxlnum) {
3265         if ($qxlnum > 1) {
3266             if ($winversion){
3267                 for(my $i = 1; $i < $qxlnum; $i++){
3268                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3269                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3270                 }
3271             } else {
3272                 # assume other OS works like Linux
3273                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3274                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3275             }
3276         }
3277
3278         my $pciaddr = print_pci_addr("spice", $bridges);
3279
3280         my $nodename = PVE::INotify::nodename();
3281         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3282         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3283         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3284         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3285         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3286
3287         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3288
3289         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3290         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3291         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3292     }
3293
3294     # enable balloon by default, unless explicitly disabled
3295     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3296         $pciaddr = print_pci_addr("balloon0", $bridges);
3297         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3298     }
3299
3300     if ($conf->{watchdog}) {
3301         my $wdopts = parse_watchdog($conf->{watchdog});
3302         $pciaddr = print_pci_addr("watchdog", $bridges);
3303         my $watchdog = $wdopts->{model} || 'i6300esb';
3304         push @$devices, '-device', "$watchdog$pciaddr";
3305         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3306     }
3307
3308     my $vollist = [];
3309     my $scsicontroller = {};
3310     my $ahcicontroller = {};
3311     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3312
3313     # Add iscsi initiator name if available
3314     if (my $initiator = get_initiator_name()) {
3315         push @$devices, '-iscsi', "initiator-name=$initiator";
3316     }
3317
3318     foreach_drive($conf, sub {
3319         my ($ds, $drive) = @_;
3320
3321         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3322             push @$vollist, $drive->{file};
3323         }
3324
3325         # ignore efidisk here, already added in bios/fw handling code above
3326         return if $drive->{interface} eq 'efidisk';
3327
3328         $use_virtio = 1 if $ds =~ m/^virtio/;
3329
3330         if (drive_is_cdrom ($drive)) {
3331             if ($bootindex_hash->{d}) {
3332                 $drive->{bootindex} = $bootindex_hash->{d};
3333                 $bootindex_hash->{d} += 1;
3334             }
3335         } else {
3336             if ($bootindex_hash->{c}) {
3337                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3338                 $bootindex_hash->{c} += 1;
3339             }
3340         }
3341
3342         if($drive->{interface} eq 'virtio'){
3343            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3344         }
3345
3346         if ($drive->{interface} eq 'scsi') {
3347
3348             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3349
3350             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3351             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3352
3353             my $iothread = '';
3354             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3355                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3356                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3357             } elsif ($drive->{iothread}) {
3358                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3359             }
3360
3361             my $queues = '';
3362             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3363                 $queues = ",num_queues=$drive->{queues}";
3364             }
3365
3366             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3367             $scsicontroller->{$controller}=1;
3368         }
3369
3370         if ($drive->{interface} eq 'sata') {
3371            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3372            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3373            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3374            $ahcicontroller->{$controller}=1;
3375         }
3376
3377         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3378         push @$devices, '-drive',$drive_cmd;
3379         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3380     });
3381
3382     for (my $i = 0; $i < $MAX_NETS; $i++) {
3383          next if !$conf->{"net$i"};
3384          my $d = parse_net($conf->{"net$i"});
3385          next if !$d;
3386
3387          $use_virtio = 1 if $d->{model} eq 'virtio';
3388
3389          if ($bootindex_hash->{n}) {
3390             $d->{bootindex} = $bootindex_hash->{n};
3391             $bootindex_hash->{n} += 1;
3392          }
3393
3394          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3395          push @$devices, '-netdev', $netdevfull;
3396
3397          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3398          push @$devices, '-device', $netdevicefull;
3399     }
3400
3401     if (!$q35) {
3402         # add pci bridges
3403         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3404            $bridges->{1} = 1;
3405            $bridges->{2} = 1;
3406         }
3407
3408         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3409
3410         while (my ($k, $v) = each %$bridges) {
3411             $pciaddr = print_pci_addr("pci.$k");
3412             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3413         }
3414     }
3415
3416     # add custom args
3417     if ($conf->{args}) {
3418         my $aa = PVE::Tools::split_args($conf->{args});
3419         push @$cmd, @$aa;
3420     }
3421
3422     push @$cmd, @$devices;
3423     push @$cmd, '-rtc', join(',', @$rtcFlags)
3424         if scalar(@$rtcFlags);
3425     push @$cmd, '-machine', join(',', @$machineFlags)
3426         if scalar(@$machineFlags);
3427     push @$cmd, '-global', join(',', @$globalFlags)
3428         if scalar(@$globalFlags);
3429
3430     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3431 }
3432
3433 sub vnc_socket {
3434     my ($vmid) = @_;
3435     return "${var_run_tmpdir}/$vmid.vnc";
3436 }
3437
3438 sub spice_port {
3439     my ($vmid) = @_;
3440
3441     my $res = vm_mon_cmd($vmid, 'query-spice');
3442
3443     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3444 }
3445
3446 sub qmp_socket {
3447     my ($vmid, $qga) = @_;
3448     my $sockettype = $qga ? 'qga' : 'qmp';
3449     return "${var_run_tmpdir}/$vmid.$sockettype";
3450 }
3451
3452 sub pidfile_name {
3453     my ($vmid) = @_;
3454     return "${var_run_tmpdir}/$vmid.pid";
3455 }
3456
3457 sub vm_devices_list {
3458     my ($vmid) = @_;
3459
3460     my $res = vm_mon_cmd($vmid, 'query-pci');
3461     my $devices = {};
3462     foreach my $pcibus (@$res) {
3463         foreach my $device (@{$pcibus->{devices}}) {
3464             next if !$device->{'qdev_id'};
3465             if ($device->{'pci_bridge'}) {
3466                 $devices->{$device->{'qdev_id'}} = 1;
3467                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3468                     next if !$bridge_device->{'qdev_id'};
3469                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3470                     $devices->{$device->{'qdev_id'}}++;
3471                 }
3472             } else {
3473                 $devices->{$device->{'qdev_id'}} = 1;
3474             }
3475         }
3476     }
3477
3478     my $resblock = vm_mon_cmd($vmid, 'query-block');
3479     foreach my $block (@$resblock) {
3480         if($block->{device} =~ m/^drive-(\S+)/){
3481                 $devices->{$1} = 1;
3482         }
3483     }
3484
3485     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3486     foreach my $mice (@$resmice) {
3487         if ($mice->{name} eq 'QEMU HID Tablet') {
3488             $devices->{tablet} = 1;
3489             last;
3490         }
3491     }
3492
3493     # for usb devices there is no query-usb
3494     # but we can iterate over the entries in
3495     # qom-list path=/machine/peripheral
3496     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3497     foreach my $per (@$resperipheral) {
3498         if ($per->{name} =~ m/^usb\d+$/) {
3499             $devices->{$per->{name}} = 1;
3500         }
3501     }
3502
3503     return $devices;
3504 }
3505
3506 sub vm_deviceplug {
3507     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3508
3509     my $q35 = machine_type_is_q35($conf);
3510
3511     my $devices_list = vm_devices_list($vmid);
3512     return 1 if defined($devices_list->{$deviceid});
3513
3514     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3515
3516     if ($deviceid eq 'tablet') {
3517
3518         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3519
3520     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3521
3522         die "usb hotplug currently not reliable\n";
3523         # since we can't reliably hot unplug all added usb devices
3524         # and usb passthrough disables live migration
3525         # we disable usb hotplugging for now
3526         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3527
3528     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3529
3530         qemu_iothread_add($vmid, $deviceid, $device);
3531
3532         qemu_driveadd($storecfg, $vmid, $device);
3533         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3534
3535         qemu_deviceadd($vmid, $devicefull);
3536         eval { qemu_deviceaddverify($vmid, $deviceid); };
3537         if (my $err = $@) {
3538             eval { qemu_drivedel($vmid, $deviceid); };
3539             warn $@ if $@;
3540             die $err;
3541         }
3542
3543     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3544
3545
3546         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3547         my $pciaddr = print_pci_addr($deviceid);
3548         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3549
3550         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3551
3552         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3553             qemu_iothread_add($vmid, $deviceid, $device);
3554             $devicefull .= ",iothread=iothread-$deviceid";
3555         }
3556
3557         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3558             $devicefull .= ",num_queues=$device->{queues}";
3559         }
3560
3561         qemu_deviceadd($vmid, $devicefull);
3562         qemu_deviceaddverify($vmid, $deviceid);
3563
3564     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3565
3566         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3567         qemu_driveadd($storecfg, $vmid, $device);
3568
3569         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3570         eval { qemu_deviceadd($vmid, $devicefull); };
3571         if (my $err = $@) {
3572             eval { qemu_drivedel($vmid, $deviceid); };
3573             warn $@ if $@;
3574             die $err;
3575         }
3576
3577     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3578
3579         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3580
3581         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3582         my $use_old_bios_files = undef;
3583         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3584
3585         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3586         qemu_deviceadd($vmid, $netdevicefull);
3587         eval { qemu_deviceaddverify($vmid, $deviceid); };
3588         if (my $err = $@) {
3589             eval { qemu_netdevdel($vmid, $deviceid); };
3590             warn $@ if $@;
3591             die $err;
3592         }
3593
3594     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3595
3596         my $bridgeid = $2;
3597         my $pciaddr = print_pci_addr($deviceid);
3598         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3599
3600         qemu_deviceadd($vmid, $devicefull);
3601         qemu_deviceaddverify($vmid, $deviceid);
3602
3603     } else {
3604         die "can't hotplug device '$deviceid'\n";
3605     }
3606
3607     return 1;
3608 }
3609
3610 # fixme: this should raise exceptions on error!
3611 sub vm_deviceunplug {
3612     my ($vmid, $conf, $deviceid) = @_;
3613
3614     my $devices_list = vm_devices_list($vmid);
3615     return 1 if !defined($devices_list->{$deviceid});
3616
3617     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3618
3619     if ($deviceid eq 'tablet') {
3620
3621         qemu_devicedel($vmid, $deviceid);
3622
3623     } elsif ($deviceid =~ m/^usb\d+$/) {
3624
3625         die "usb hotplug currently not reliable\n";
3626         # when unplugging usb devices this way,
3627         # there may be remaining usb controllers/hubs
3628         # so we disable it for now
3629         qemu_devicedel($vmid, $deviceid);
3630         qemu_devicedelverify($vmid, $deviceid);
3631
3632     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3633
3634         qemu_devicedel($vmid, $deviceid);
3635         qemu_devicedelverify($vmid, $deviceid);
3636         qemu_drivedel($vmid, $deviceid);
3637         qemu_iothread_del($conf, $vmid, $deviceid);
3638
3639     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3640
3641         qemu_devicedel($vmid, $deviceid);
3642         qemu_devicedelverify($vmid, $deviceid);
3643         qemu_iothread_del($conf, $vmid, $deviceid);
3644
3645     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3646
3647         qemu_devicedel($vmid, $deviceid);
3648         qemu_drivedel($vmid, $deviceid);
3649         qemu_deletescsihw($conf, $vmid, $deviceid);
3650
3651     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3652
3653         qemu_devicedel($vmid, $deviceid);
3654         qemu_devicedelverify($vmid, $deviceid);
3655         qemu_netdevdel($vmid, $deviceid);
3656
3657     } else {
3658         die "can't unplug device '$deviceid'\n";
3659     }
3660
3661     return 1;
3662 }
3663
3664 sub qemu_deviceadd {
3665     my ($vmid, $devicefull) = @_;
3666
3667     $devicefull = "driver=".$devicefull;
3668     my %options =  split(/[=,]/, $devicefull);
3669
3670     vm_mon_cmd($vmid, "device_add" , %options);
3671 }
3672
3673 sub qemu_devicedel {
3674     my ($vmid, $deviceid) = @_;
3675
3676     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3677 }
3678
3679 sub qemu_iothread_add {
3680     my($vmid, $deviceid, $device) = @_;
3681
3682     if ($device->{iothread}) {
3683         my $iothreads = vm_iothreads_list($vmid);
3684         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3685     }
3686 }
3687
3688 sub qemu_iothread_del {
3689     my($conf, $vmid, $deviceid) = @_;
3690
3691     my $device = parse_drive($deviceid, $conf->{$deviceid});
3692     if ($device->{iothread}) {
3693         my $iothreads = vm_iothreads_list($vmid);
3694         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3695     }
3696 }
3697
3698 sub qemu_objectadd {
3699     my($vmid, $objectid, $qomtype) = @_;
3700
3701     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3702
3703     return 1;
3704 }
3705
3706 sub qemu_objectdel {
3707     my($vmid, $objectid) = @_;
3708
3709     vm_mon_cmd($vmid, "object-del", id => $objectid);
3710
3711     return 1;
3712 }
3713
3714 sub qemu_driveadd {
3715     my ($storecfg, $vmid, $device) = @_;
3716
3717     my $drive = print_drive_full($storecfg, $vmid, $device);
3718     $drive =~ s/\\/\\\\/g;
3719     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3720
3721     # If the command succeeds qemu prints: "OK"
3722     return 1 if $ret =~ m/OK/s;
3723
3724     die "adding drive failed: $ret\n";
3725 }
3726
3727 sub qemu_drivedel {
3728     my($vmid, $deviceid) = @_;
3729
3730     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3731     $ret =~ s/^\s+//;
3732
3733     return 1 if $ret eq "";
3734
3735     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3736     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3737
3738     die "deleting drive $deviceid failed : $ret\n";
3739 }
3740
3741 sub qemu_deviceaddverify {
3742     my ($vmid, $deviceid) = @_;
3743
3744     for (my $i = 0; $i <= 5; $i++) {
3745          my $devices_list = vm_devices_list($vmid);
3746          return 1 if defined($devices_list->{$deviceid});
3747          sleep 1;
3748     }
3749
3750     die "error on hotplug device '$deviceid'\n";
3751 }
3752
3753
3754 sub qemu_devicedelverify {
3755     my ($vmid, $deviceid) = @_;
3756
3757     # need to verify that the device is correctly removed as device_del
3758     # is async and empty return is not reliable
3759
3760     for (my $i = 0; $i <= 5; $i++) {
3761          my $devices_list = vm_devices_list($vmid);
3762          return 1 if !defined($devices_list->{$deviceid});
3763          sleep 1;
3764     }
3765
3766     die "error on hot-unplugging device '$deviceid'\n";
3767 }
3768
3769 sub qemu_findorcreatescsihw {
3770     my ($storecfg, $conf, $vmid, $device) = @_;
3771
3772     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3773
3774     my $scsihwid="$controller_prefix$controller";
3775     my $devices_list = vm_devices_list($vmid);
3776
3777     if(!defined($devices_list->{$scsihwid})) {
3778         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3779     }
3780
3781     return 1;
3782 }
3783
3784 sub qemu_deletescsihw {
3785     my ($conf, $vmid, $opt) = @_;
3786
3787     my $device = parse_drive($opt, $conf->{$opt});
3788
3789     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3790         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3791         return 1;
3792     }
3793
3794     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3795
3796     my $devices_list = vm_devices_list($vmid);
3797     foreach my $opt (keys %{$devices_list}) {
3798         if (PVE::QemuServer::is_valid_drivename($opt)) {
3799             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3800             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3801                 return 1;
3802             }
3803         }
3804     }
3805
3806     my $scsihwid="scsihw$controller";
3807
3808     vm_deviceunplug($vmid, $conf, $scsihwid);
3809
3810     return 1;
3811 }
3812
3813 sub qemu_add_pci_bridge {
3814     my ($storecfg, $conf, $vmid, $device) = @_;
3815
3816     my $bridges = {};
3817
3818     my $bridgeid;
3819
3820     print_pci_addr($device, $bridges);
3821
3822     while (my ($k, $v) = each %$bridges) {
3823         $bridgeid = $k;
3824     }
3825     return 1 if !defined($bridgeid) || $bridgeid < 1;
3826
3827     my $bridge = "pci.$bridgeid";
3828     my $devices_list = vm_devices_list($vmid);
3829
3830     if (!defined($devices_list->{$bridge})) {
3831         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3832     }
3833
3834     return 1;
3835 }
3836
3837 sub qemu_set_link_status {
3838     my ($vmid, $device, $up) = @_;
3839
3840     vm_mon_cmd($vmid, "set_link", name => $device,
3841                up => $up ? JSON::true : JSON::false);
3842 }
3843
3844 sub qemu_netdevadd {
3845     my ($vmid, $conf, $device, $deviceid) = @_;
3846
3847     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3848     my %options =  split(/[=,]/, $netdev);
3849
3850     vm_mon_cmd($vmid, "netdev_add",  %options);
3851     return 1;
3852 }
3853
3854 sub qemu_netdevdel {
3855     my ($vmid, $deviceid) = @_;
3856
3857     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3858 }
3859
3860 sub qemu_usb_hotplug {
3861     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3862
3863     return if !$device;
3864
3865     # remove the old one first
3866     vm_deviceunplug($vmid, $conf, $deviceid);
3867
3868     # check if xhci controller is necessary and available
3869     if ($device->{usb3}) {
3870
3871         my $devicelist = vm_devices_list($vmid);
3872
3873         if (!$devicelist->{xhci}) {
3874             my $pciaddr = print_pci_addr("xhci");
3875             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
3876         }
3877     }
3878     my $d = parse_usb_device($device->{host});
3879     $d->{usb3} = $device->{usb3};
3880
3881     # add the new one
3882     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
3883 }
3884
3885 sub qemu_cpu_hotplug {
3886     my ($vmid, $conf, $vcpus) = @_;
3887
3888     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
3889
3890     my $sockets = 1;
3891     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3892     $sockets = $conf->{sockets} if  $conf->{sockets};
3893     my $cores = $conf->{cores} || 1;
3894     my $maxcpus = $sockets * $cores;
3895
3896     $vcpus = $maxcpus if !$vcpus;
3897
3898     die "you can't add more vcpus than maxcpus\n"
3899         if $vcpus > $maxcpus;
3900
3901     my $currentvcpus = $conf->{vcpus} || $maxcpus;
3902
3903     if ($vcpus < $currentvcpus) {
3904
3905         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3906
3907             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
3908                 qemu_devicedel($vmid, "cpu$i");
3909                 my $retry = 0;
3910                 my $currentrunningvcpus = undef;
3911                 while (1) {
3912                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3913                     last if scalar(@{$currentrunningvcpus}) == $i-1;
3914                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
3915                     $retry++;
3916                     sleep 1;
3917                 }
3918                 #update conf after each succesfull cpu unplug
3919                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3920                 PVE::QemuConfig->write_config($vmid, $conf);
3921             }
3922         } else {
3923             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
3924         }
3925
3926         return;
3927     }
3928
3929     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3930     die "vcpus in running vm does not match its configuration\n"
3931         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
3932
3933     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
3934
3935         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
3936             my $cpustr = print_cpu_device($conf, $i);
3937             qemu_deviceadd($vmid, $cpustr);
3938
3939             my $retry = 0;
3940             my $currentrunningvcpus = undef;
3941             while (1) {
3942                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
3943                 last if scalar(@{$currentrunningvcpus}) == $i;
3944                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
3945                 sleep 1;
3946                 $retry++;
3947             }
3948             #update conf after each succesfull cpu hotplug
3949             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
3950             PVE::QemuConfig->write_config($vmid, $conf);
3951         }
3952     } else {
3953
3954         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
3955             vm_mon_cmd($vmid, "cpu-add", id => int($i));
3956         }
3957     }
3958 }
3959
3960 sub qemu_block_set_io_throttle {
3961     my ($vmid, $deviceid,
3962         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
3963         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
3964         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
3965         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
3966
3967     return if !check_running($vmid) ;
3968
3969     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
3970         bps => int($bps),
3971         bps_rd => int($bps_rd),
3972         bps_wr => int($bps_wr),
3973         iops => int($iops),
3974         iops_rd => int($iops_rd),
3975         iops_wr => int($iops_wr),
3976         bps_max => int($bps_max),
3977         bps_rd_max => int($bps_rd_max),
3978         bps_wr_max => int($bps_wr_max),
3979         iops_max => int($iops_max),
3980         iops_rd_max => int($iops_rd_max),
3981         iops_wr_max => int($iops_wr_max),
3982         bps_max_length => int($bps_max_length),
3983         bps_rd_max_length => int($bps_rd_max_length),
3984         bps_wr_max_length => int($bps_wr_max_length),
3985         iops_max_length => int($iops_max_length),
3986         iops_rd_max_length => int($iops_rd_max_length),
3987         iops_wr_max_length => int($iops_wr_max_length),
3988     );
3989
3990 }
3991
3992 # old code, only used to shutdown old VM after update
3993 sub __read_avail {
3994     my ($fh, $timeout) = @_;
3995
3996     my $sel = new IO::Select;
3997     $sel->add($fh);
3998
3999     my $res = '';
4000     my $buf;
4001
4002     my @ready;
4003     while (scalar (@ready = $sel->can_read($timeout))) {
4004         my $count;
4005         if ($count = $fh->sysread($buf, 8192)) {
4006             if ($buf =~ /^(.*)\(qemu\) $/s) {
4007                 $res .= $1;
4008                 last;
4009             } else {
4010                 $res .= $buf;
4011             }
4012         } else {
4013             if (!defined($count)) {
4014                 die "$!\n";
4015             }
4016             last;
4017         }
4018     }
4019
4020     die "monitor read timeout\n" if !scalar(@ready);
4021
4022     return $res;
4023 }
4024
4025 # old code, only used to shutdown old VM after update
4026 sub vm_monitor_command {
4027     my ($vmid, $cmdstr, $nocheck) = @_;
4028
4029     my $res;
4030
4031     eval {
4032         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4033
4034         my $sname = "${var_run_tmpdir}/$vmid.mon";
4035
4036         my $sock = IO::Socket::UNIX->new( Peer => $sname ) ||
4037             die "unable to connect to VM $vmid socket - $!\n";
4038
4039         my $timeout = 3;
4040
4041         # hack: migrate sometime blocks the monitor (when migrate_downtime
4042         # is set)
4043         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4044             $timeout = 60*60; # 1 hour
4045         }
4046
4047         # read banner;
4048         my $data = __read_avail($sock, $timeout);
4049
4050         if ($data !~ m/^QEMU\s+(\S+)\s+monitor\s/) {
4051             die "got unexpected qemu monitor banner\n";
4052         }
4053
4054         my $sel = new IO::Select;
4055         $sel->add($sock);
4056
4057         if (!scalar(my @ready = $sel->can_write($timeout))) {
4058             die "monitor write error - timeout";
4059         }
4060
4061         my $fullcmd = "$cmdstr\r";
4062
4063         # syslog('info', "VM $vmid monitor command: $cmdstr");
4064
4065         my $b;
4066         if (!($b = $sock->syswrite($fullcmd)) || ($b != length($fullcmd))) {
4067             die "monitor write error - $!";
4068         }
4069
4070         return if ($cmdstr eq 'q') || ($cmdstr eq 'quit');
4071
4072         $timeout = 20;
4073
4074         if ($cmdstr =~ m/^(info\s+migrate|migrate\s)/) {
4075             $timeout = 60*60; # 1 hour
4076         } elsif ($cmdstr =~ m/^(eject|change)/) {
4077             $timeout = 60; # note: cdrom mount command is slow
4078         }
4079         if ($res = __read_avail($sock, $timeout)) {
4080
4081             my @lines = split("\r?\n", $res);
4082
4083             shift @lines if $lines[0] !~ m/^unknown command/; # skip echo
4084
4085             $res = join("\n", @lines);
4086             $res .= "\n";
4087         }
4088     };
4089
4090     my $err = $@;
4091
4092     if ($err) {
4093         syslog("err", "VM $vmid monitor command failed - $err");
4094         die $err;
4095     }
4096
4097     return $res;
4098 }
4099
4100 sub qemu_block_resize {
4101     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4102
4103     my $running = check_running($vmid);
4104
4105     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4106
4107     return if !$running;
4108
4109     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4110
4111 }
4112
4113 sub qemu_volume_snapshot {
4114     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4115
4116     my $running = check_running($vmid);
4117
4118     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4119         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4120     } else {
4121         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4122     }
4123 }
4124
4125 sub qemu_volume_snapshot_delete {
4126     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4127
4128     my $running = check_running($vmid);
4129
4130     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4131         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4132     } else {
4133         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4134     }
4135 }
4136
4137 sub set_migration_caps {
4138     my ($vmid) = @_;
4139
4140     my $cap_ref = [];
4141
4142     my $enabled_cap = {
4143         "auto-converge" => 1,
4144         "xbzrle" => 1,
4145         "x-rdma-pin-all" => 0,
4146         "zero-blocks" => 0,
4147         "compress" => 0
4148     };
4149
4150     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4151
4152     for my $supported_capability (@$supported_capabilities) {
4153         push @$cap_ref, {
4154             capability => $supported_capability->{capability},
4155             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4156         };
4157     }
4158
4159     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4160 }
4161
4162 my $fast_plug_option = {
4163     'lock' => 1,
4164     'name' => 1,
4165     'onboot' => 1,
4166     'shares' => 1,
4167     'startup' => 1,
4168     'description' => 1,
4169     'protection' => 1,
4170     'vmstatestorage' => 1,
4171 };
4172
4173 # hotplug changes in [PENDING]
4174 # $selection hash can be used to only apply specified options, for
4175 # example: { cores => 1 } (only apply changed 'cores')
4176 # $errors ref is used to return error messages
4177 sub vmconfig_hotplug_pending {
4178     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4179
4180     my $defaults = load_defaults();
4181
4182     # commit values which do not have any impact on running VM first
4183     # Note: those option cannot raise errors, we we do not care about
4184     # $selection and always apply them.
4185
4186     my $add_error = sub {
4187         my ($opt, $msg) = @_;
4188         $errors->{$opt} = "hotplug problem - $msg";
4189     };
4190
4191     my $changes = 0;
4192     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4193         if ($fast_plug_option->{$opt}) {
4194             $conf->{$opt} = $conf->{pending}->{$opt};
4195             delete $conf->{pending}->{$opt};
4196             $changes = 1;
4197         }
4198     }
4199
4200     if ($changes) {
4201         PVE::QemuConfig->write_config($vmid, $conf);
4202         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4203     }
4204
4205     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4206
4207     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4208     while (my ($opt, $force) = each %$pending_delete_hash) {
4209         next if $selection && !$selection->{$opt};
4210         eval {
4211             if ($opt eq 'hotplug') {
4212                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4213             } elsif ($opt eq 'tablet') {
4214                 die "skip\n" if !$hotplug_features->{usb};
4215                 if ($defaults->{tablet}) {
4216                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4217                 } else {
4218                     vm_deviceunplug($vmid, $conf, $opt);
4219                 }
4220             } elsif ($opt =~ m/^usb\d+/) {
4221                 die "skip\n";
4222                 # since we cannot reliably hot unplug usb devices
4223                 # we are disabling it
4224                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4225                 vm_deviceunplug($vmid, $conf, $opt);
4226             } elsif ($opt eq 'vcpus') {
4227                 die "skip\n" if !$hotplug_features->{cpu};
4228                 qemu_cpu_hotplug($vmid, $conf, undef);
4229             } elsif ($opt eq 'balloon') {
4230                 # enable balloon device is not hotpluggable
4231                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4232             } elsif ($fast_plug_option->{$opt}) {
4233                 # do nothing
4234             } elsif ($opt =~ m/^net(\d+)$/) {
4235                 die "skip\n" if !$hotplug_features->{network};
4236                 vm_deviceunplug($vmid, $conf, $opt);
4237             } elsif (is_valid_drivename($opt)) {
4238                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4239                 vm_deviceunplug($vmid, $conf, $opt);
4240                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4241             } elsif ($opt =~ m/^memory$/) {
4242                 die "skip\n" if !$hotplug_features->{memory};
4243                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4244             } elsif ($opt eq 'cpuunits') {
4245                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4246             } elsif ($opt eq 'cpulimit') {
4247                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4248             } else {
4249                 die "skip\n";
4250             }
4251         };
4252         if (my $err = $@) {
4253             &$add_error($opt, $err) if $err ne "skip\n";
4254         } else {
4255             # save new config if hotplug was successful
4256             delete $conf->{$opt};
4257             vmconfig_undelete_pending_option($conf, $opt);
4258             PVE::QemuConfig->write_config($vmid, $conf);
4259             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4260         }
4261     }
4262
4263     foreach my $opt (keys %{$conf->{pending}}) {
4264         next if $selection && !$selection->{$opt};
4265         my $value = $conf->{pending}->{$opt};
4266         eval {
4267             if ($opt eq 'hotplug') {
4268                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4269             } elsif ($opt eq 'tablet') {
4270                 die "skip\n" if !$hotplug_features->{usb};
4271                 if ($value == 1) {
4272                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4273                 } elsif ($value == 0) {
4274                     vm_deviceunplug($vmid, $conf, $opt);
4275                 }
4276             } elsif ($opt =~ m/^usb\d+$/) {
4277                 die "skip\n";
4278                 # since we cannot reliably hot unplug usb devices
4279                 # we are disabling it
4280                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4281                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4282                 die "skip\n" if !$d;
4283                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4284             } elsif ($opt eq 'vcpus') {
4285                 die "skip\n" if !$hotplug_features->{cpu};
4286                 qemu_cpu_hotplug($vmid, $conf, $value);
4287             } elsif ($opt eq 'balloon') {
4288                 # enable/disable balloning device is not hotpluggable
4289                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4290                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4291                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4292
4293                 # allow manual ballooning if shares is set to zero
4294                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4295                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4296                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4297                 }
4298             } elsif ($opt =~ m/^net(\d+)$/) {
4299                 # some changes can be done without hotplug
4300                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4301                                     $vmid, $opt, $value);
4302             } elsif (is_valid_drivename($opt)) {
4303                 # some changes can be done without hotplug
4304                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4305                                      $vmid, $opt, $value, 1);
4306             } elsif ($opt =~ m/^memory$/) { #dimms
4307                 die "skip\n" if !$hotplug_features->{memory};
4308                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4309             } elsif ($opt eq 'cpuunits') {
4310                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4311             } elsif ($opt eq 'cpulimit') {
4312                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4313                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4314             } else {
4315                 die "skip\n";  # skip non-hot-pluggable options
4316             }
4317         };
4318         if (my $err = $@) {
4319             &$add_error($opt, $err) if $err ne "skip\n";
4320         } else {
4321             # save new config if hotplug was successful
4322             $conf->{$opt} = $value;
4323             delete $conf->{pending}->{$opt};
4324             PVE::QemuConfig->write_config($vmid, $conf);
4325             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4326         }
4327     }
4328 }
4329
4330 sub try_deallocate_drive {
4331     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4332
4333     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4334         my $volid = $drive->{file};
4335         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4336             my $sid = PVE::Storage::parse_volume_id($volid);
4337             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4338
4339             # check if the disk is really unused
4340             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4341                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4342             PVE::Storage::vdisk_free($storecfg, $volid);
4343             return 1;
4344         } else {
4345             # If vm is not owner of this disk remove from config
4346             return 1;
4347         }
4348     }
4349
4350     return undef;
4351 }
4352
4353 sub vmconfig_delete_or_detach_drive {
4354     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4355
4356     my $drive = parse_drive($opt, $conf->{$opt});
4357
4358     my $rpcenv = PVE::RPCEnvironment::get();
4359     my $authuser = $rpcenv->get_user();
4360
4361     if ($force) {
4362         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4363         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4364     } else {
4365         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4366     }
4367 }
4368
4369 sub vmconfig_apply_pending {
4370     my ($vmid, $conf, $storecfg) = @_;
4371
4372     # cold plug
4373
4374     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4375     while (my ($opt, $force) = each %$pending_delete_hash) {
4376         die "internal error" if $opt =~ m/^unused/;
4377         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4378         if (!defined($conf->{$opt})) {
4379             vmconfig_undelete_pending_option($conf, $opt);
4380             PVE::QemuConfig->write_config($vmid, $conf);
4381         } elsif (is_valid_drivename($opt)) {
4382             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4383             vmconfig_undelete_pending_option($conf, $opt);
4384             delete $conf->{$opt};
4385             PVE::QemuConfig->write_config($vmid, $conf);
4386         } else {
4387             vmconfig_undelete_pending_option($conf, $opt);
4388             delete $conf->{$opt};
4389             PVE::QemuConfig->write_config($vmid, $conf);
4390         }
4391     }
4392
4393     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4394
4395     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4396         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4397
4398         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4399             # skip if nothing changed
4400         } elsif (is_valid_drivename($opt)) {
4401             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4402                 if defined($conf->{$opt});
4403             $conf->{$opt} = $conf->{pending}->{$opt};
4404         } else {
4405             $conf->{$opt} = $conf->{pending}->{$opt};
4406         }
4407
4408         delete $conf->{pending}->{$opt};
4409         PVE::QemuConfig->write_config($vmid, $conf);
4410     }
4411 }
4412
4413 my $safe_num_ne = sub {
4414     my ($a, $b) = @_;
4415
4416     return 0 if !defined($a) && !defined($b);
4417     return 1 if !defined($a);
4418     return 1 if !defined($b);
4419
4420     return $a != $b;
4421 };
4422
4423 my $safe_string_ne = sub {
4424     my ($a, $b) = @_;
4425
4426     return 0 if !defined($a) && !defined($b);
4427     return 1 if !defined($a);
4428     return 1 if !defined($b);
4429
4430     return $a ne $b;
4431 };
4432
4433 sub vmconfig_update_net {
4434     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4435
4436     my $newnet = parse_net($value);
4437
4438     if ($conf->{$opt}) {
4439         my $oldnet = parse_net($conf->{$opt});
4440
4441         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4442             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4443             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4444             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4445
4446             # for non online change, we try to hot-unplug
4447             die "skip\n" if !$hotplug;
4448             vm_deviceunplug($vmid, $conf, $opt);
4449         } else {
4450
4451             die "internal error" if $opt !~ m/net(\d+)/;
4452             my $iface = "tap${vmid}i$1";
4453
4454             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4455                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4456                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4457                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4458                 PVE::Network::tap_unplug($iface);
4459                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4460             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4461                 # Rate can be applied on its own but any change above needs to
4462                 # include the rate in tap_plug since OVS resets everything.
4463                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4464             }
4465
4466             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4467                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4468             }
4469
4470             return 1;
4471         }
4472     }
4473
4474     if ($hotplug) {
4475         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4476     } else {
4477         die "skip\n";
4478     }
4479 }
4480
4481 sub vmconfig_update_disk {
4482     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4483
4484     # fixme: do we need force?
4485
4486     my $drive = parse_drive($opt, $value);
4487
4488     if ($conf->{$opt}) {
4489
4490         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4491
4492             my $media = $drive->{media} || 'disk';
4493             my $oldmedia = $old_drive->{media} || 'disk';
4494             die "unable to change media type\n" if $media ne $oldmedia;
4495
4496             if (!drive_is_cdrom($old_drive)) {
4497
4498                 if ($drive->{file} ne $old_drive->{file}) {
4499
4500                     die "skip\n" if !$hotplug;
4501
4502                     # unplug and register as unused
4503                     vm_deviceunplug($vmid, $conf, $opt);
4504                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4505
4506                 } else {
4507                     # update existing disk
4508
4509                     # skip non hotpluggable value
4510                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4511                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4512                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4513                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4514                         die "skip\n";
4515                     }
4516
4517                     # apply throttle
4518                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4519                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4520                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4521                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4522                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4523                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4524                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4525                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4526                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4527                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4528                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4529                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4530                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4531                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4532                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4533                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4534                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4535                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4536
4537                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4538                                                    ($drive->{mbps} || 0)*1024*1024,
4539                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4540                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4541                                                    $drive->{iops} || 0,
4542                                                    $drive->{iops_rd} || 0,
4543                                                    $drive->{iops_wr} || 0,
4544                                                    ($drive->{mbps_max} || 0)*1024*1024,
4545                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4546                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4547                                                    $drive->{iops_max} || 0,
4548                                                    $drive->{iops_rd_max} || 0,
4549                                                    $drive->{iops_wr_max} || 0,
4550                                                    $drive->{bps_max_length} || 1,
4551                                                    $drive->{bps_rd_max_length} || 1,
4552                                                    $drive->{bps_wr_max_length} || 1,
4553                                                    $drive->{iops_max_length} || 1,
4554                                                    $drive->{iops_rd_max_length} || 1,
4555                                                    $drive->{iops_wr_max_length} || 1);
4556
4557                     }
4558
4559                     return 1;
4560                 }
4561
4562             } else { # cdrom
4563
4564                 if ($drive->{file} eq 'none') {
4565                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4566                 } else {
4567                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4568                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4569                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4570                 }
4571
4572                 return 1;
4573             }
4574         }
4575     }
4576
4577     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4578     # hotplug new disks
4579     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4580     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4581 }
4582
4583 sub vm_start {
4584     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4585         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4586
4587     PVE::QemuConfig->lock_config($vmid, sub {
4588         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4589
4590         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4591
4592         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4593
4594         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4595
4596         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4597             vmconfig_apply_pending($vmid, $conf, $storecfg);
4598             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4599         }
4600
4601         my $defaults = load_defaults();
4602
4603         # set environment variable useful inside network script
4604         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4605
4606         my $local_volumes = {};
4607
4608         if ($targetstorage) {
4609             foreach_drive($conf, sub {
4610                 my ($ds, $drive) = @_;
4611
4612                 return if drive_is_cdrom($drive);
4613
4614                 my $volid = $drive->{file};
4615
4616                 return if !$volid;
4617
4618                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4619
4620                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4621                 return if $scfg->{shared};
4622                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4623             });
4624
4625             my $format = undef;
4626
4627             foreach my $opt (sort keys %$local_volumes) {
4628
4629                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4630                 my $drive = parse_drive($opt, $conf->{$opt});
4631
4632                 #if remote storage is specified, use default format
4633                 if ($targetstorage && $targetstorage ne "1") {
4634                     $storeid = $targetstorage;
4635                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4636                     $format = $defFormat;
4637                 } else {
4638                     #else we use same format than original
4639                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4640                     $format = qemu_img_format($scfg, $volid);
4641                 }
4642
4643                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4644                 my $newdrive = $drive;
4645                 $newdrive->{format} = $format;
4646                 $newdrive->{file} = $newvolid;
4647                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4648                 $local_volumes->{$opt} = $drivestr;
4649                 #pass drive to conf for command line
4650                 $conf->{$opt} = $drivestr;
4651             }
4652         }
4653
4654         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4655
4656         my $migrate_port = 0;
4657         my $migrate_uri;
4658         if ($statefile) {
4659             if ($statefile eq 'tcp') {
4660                 my $localip = "localhost";
4661                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4662                 my $nodename = PVE::INotify::nodename();
4663
4664                 if (!defined($migration_type)) {
4665                     if (defined($datacenterconf->{migration}->{type})) {
4666                         $migration_type = $datacenterconf->{migration}->{type};
4667                     } else {
4668                         $migration_type = 'secure';
4669                     }
4670                 }
4671
4672                 if ($migration_type eq 'insecure') {
4673                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4674                     if ($migrate_network_addr) {
4675                         $localip = $migrate_network_addr;
4676                     } else {
4677                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4678                     }
4679
4680                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4681                 }
4682
4683                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4684                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4685                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4686                 push @$cmd, '-incoming', $migrate_uri;
4687                 push @$cmd, '-S';
4688
4689             } elsif ($statefile eq 'unix') {
4690                 # should be default for secure migrations as a ssh TCP forward
4691                 # tunnel is not deterministic reliable ready and fails regurarly
4692                 # to set up in time, so use UNIX socket forwards
4693                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4694                 unlink $socket_addr;
4695
4696                 $migrate_uri = "unix:$socket_addr";
4697
4698                 push @$cmd, '-incoming', $migrate_uri;
4699                 push @$cmd, '-S';
4700
4701             } else {
4702                 push @$cmd, '-loadstate', $statefile;
4703             }
4704         } elsif ($paused) {
4705             push @$cmd, '-S';
4706         }
4707
4708         # host pci devices
4709         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4710           my $d = parse_hostpci($conf->{"hostpci$i"});
4711           next if !$d;
4712           my $pcidevices = $d->{pciid};
4713           foreach my $pcidevice (@$pcidevices) {
4714                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4715
4716                 my $info = pci_device_info("0000:$pciid");
4717                 die "IOMMU not present\n" if !check_iommu_support();
4718                 die "no pci device info for device '$pciid'\n" if !$info;
4719                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4720                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4721           }
4722         }
4723
4724         PVE::Storage::activate_volumes($storecfg, $vollist);
4725
4726         if (!check_running($vmid, 1) && -d "/sys/fs/cgroup/systemd/qemu.slice/$vmid.scope") {
4727             my $cmd = [];
4728             push @$cmd, '/bin/systemctl', 'stop', "$vmid.scope";
4729             eval  { run_command($cmd); };
4730         }
4731
4732         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4733                                                   : $defaults->{cpuunits};
4734
4735         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4736         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4737
4738         my %properties = (
4739             Slice => 'qemu.slice',
4740             KillMode => 'none',
4741             CPUShares => $cpuunits
4742         );
4743
4744         if (my $cpulimit = $conf->{cpulimit}) {
4745             $properties{CPUQuota} = int($cpulimit * 100);
4746         }
4747         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4748
4749         if ($conf->{hugepages}) {
4750
4751             my $code = sub {
4752                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4753                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4754
4755                 PVE::QemuServer::Memory::hugepages_mount();
4756                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4757
4758                 eval  {
4759                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4760                     run_command($cmd, %run_params);
4761                 };
4762
4763                 if (my $err = $@) {
4764                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4765                     die $err;
4766                 }
4767
4768                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4769             };
4770             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4771
4772         } else {
4773             eval  {
4774                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4775                 run_command($cmd, %run_params);
4776             };
4777         }
4778
4779         if (my $err = $@) {
4780             # deactivate volumes if start fails
4781             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4782             die "start failed: $err";
4783         }
4784
4785         print "migration listens on $migrate_uri\n" if $migrate_uri;
4786
4787         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4788             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4789             warn $@ if $@;
4790         }
4791
4792         #start nbd server for storage migration
4793         if ($targetstorage) {
4794             my $nodename = PVE::INotify::nodename();
4795             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4796             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4797             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4798             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4799
4800             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4801
4802             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4803
4804             foreach my $opt (sort keys %$local_volumes) {
4805                 my $volid = $local_volumes->{$opt};
4806                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4807                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4808                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4809             }
4810         }
4811
4812         if ($migratedfrom) {
4813             eval {
4814                 set_migration_caps($vmid);
4815             };
4816             warn $@ if $@;
4817
4818             if ($spice_port) {
4819                 print "spice listens on port $spice_port\n";
4820                 if ($spice_ticket) {
4821                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4822                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4823                 }
4824             }
4825
4826         } else {
4827             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4828                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4829                     if $conf->{balloon};
4830             }
4831
4832             foreach my $opt (keys %$conf) {
4833                 next if $opt !~  m/^net\d+$/;
4834                 my $nicconf = parse_net($conf->{$opt});
4835                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4836             }
4837         }
4838
4839         vm_mon_cmd_nocheck($vmid, 'qom-set',
4840                     path => "machine/peripheral/balloon0",
4841                     property => "guest-stats-polling-interval",
4842                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4843
4844     });
4845 }
4846
4847 sub vm_mon_cmd {
4848     my ($vmid, $execute, %params) = @_;
4849
4850     my $cmd = { execute => $execute, arguments => \%params };
4851     vm_qmp_command($vmid, $cmd);
4852 }
4853
4854 sub vm_mon_cmd_nocheck {
4855     my ($vmid, $execute, %params) = @_;
4856
4857     my $cmd = { execute => $execute, arguments => \%params };
4858     vm_qmp_command($vmid, $cmd, 1);
4859 }
4860
4861 sub vm_qmp_command {
4862     my ($vmid, $cmd, $nocheck) = @_;
4863
4864     my $res;
4865
4866     my $timeout;
4867     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4868         $timeout = $cmd->{arguments}->{timeout};
4869         delete $cmd->{arguments}->{timeout};
4870     }
4871
4872     eval {
4873         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4874         my $sname = qmp_socket($vmid);
4875         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4876             my $qmpclient = PVE::QMPClient->new();
4877
4878             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4879         } elsif (-e "${var_run_tmpdir}/$vmid.mon") {
4880             die "can't execute complex command on old monitor - stop/start your vm to fix the problem\n"
4881                 if scalar(%{$cmd->{arguments}});
4882             vm_monitor_command($vmid, $cmd->{execute}, $nocheck);
4883         } else {
4884             die "unable to open monitor socket\n";
4885         }
4886     };
4887     if (my $err = $@) {
4888         syslog("err", "VM $vmid qmp command failed - $err");
4889         die $err;
4890     }
4891
4892     return $res;
4893 }
4894
4895 sub vm_human_monitor_command {
4896     my ($vmid, $cmdline) = @_;
4897
4898     my $res;
4899
4900     my $cmd = {
4901         execute => 'human-monitor-command',
4902         arguments => { 'command-line' => $cmdline},
4903     };
4904
4905     return vm_qmp_command($vmid, $cmd);
4906 }
4907
4908 sub vm_commandline {
4909     my ($storecfg, $vmid) = @_;
4910
4911     my $conf = PVE::QemuConfig->load_config($vmid);
4912
4913     my $defaults = load_defaults();
4914
4915     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
4916
4917     return PVE::Tools::cmd2string($cmd);
4918 }
4919
4920 sub vm_reset {
4921     my ($vmid, $skiplock) = @_;
4922
4923     PVE::QemuConfig->lock_config($vmid, sub {
4924
4925         my $conf = PVE::QemuConfig->load_config($vmid);
4926
4927         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4928
4929         vm_mon_cmd($vmid, "system_reset");
4930     });
4931 }
4932
4933 sub get_vm_volumes {
4934     my ($conf) = @_;
4935
4936     my $vollist = [];
4937     foreach_volid($conf, sub {
4938         my ($volid, $attr) = @_;
4939
4940         return if $volid =~ m|^/|;
4941
4942         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
4943         return if !$sid;
4944
4945         push @$vollist, $volid;
4946     });
4947
4948     return $vollist;
4949 }
4950
4951 sub vm_stop_cleanup {
4952     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
4953
4954     eval {
4955
4956         if (!$keepActive) {
4957             my $vollist = get_vm_volumes($conf);
4958             PVE::Storage::deactivate_volumes($storecfg, $vollist);
4959         }
4960
4961         foreach my $ext (qw(mon qmp pid vnc qga)) {
4962             unlink "/var/run/qemu-server/${vmid}.$ext";
4963         }
4964
4965         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
4966     };
4967     warn $@ if $@; # avoid errors - just warn
4968 }
4969
4970 # Note: use $nockeck to skip tests if VM configuration file exists.
4971 # We need that when migration VMs to other nodes (files already moved)
4972 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
4973 sub vm_stop {
4974     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
4975
4976     $force = 1 if !defined($force) && !$shutdown;
4977
4978     if ($migratedfrom){
4979         my $pid = check_running($vmid, $nocheck, $migratedfrom);
4980         kill 15, $pid if $pid;
4981         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4982         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
4983         return;
4984     }
4985
4986     PVE::QemuConfig->lock_config($vmid, sub {
4987
4988         my $pid = check_running($vmid, $nocheck);
4989         return if !$pid;
4990
4991         my $conf;
4992         if (!$nocheck) {
4993             $conf = PVE::QemuConfig->load_config($vmid);
4994             PVE::QemuConfig->check_lock($conf) if !$skiplock;
4995             if (!defined($timeout) && $shutdown && $conf->{startup}) {
4996                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
4997                 $timeout = $opts->{down} if $opts->{down};
4998             }
4999         }
5000
5001         $timeout = 60 if !defined($timeout);
5002
5003         eval {
5004             if ($shutdown) {
5005                 if (defined($conf) && $conf->{agent}) {
5006                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5007                 } else {
5008                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5009                 }
5010             } else {
5011                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5012             }
5013         };
5014         my $err = $@;
5015
5016         if (!$err) {
5017             my $count = 0;
5018             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5019                 $count++;
5020                 sleep 1;
5021             }
5022
5023             if ($count >= $timeout) {
5024                 if ($force) {
5025                     warn "VM still running - terminating now with SIGTERM\n";
5026                     kill 15, $pid;
5027                 } else {
5028                     die "VM quit/powerdown failed - got timeout\n";
5029                 }
5030             } else {
5031                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5032                 return;
5033             }
5034         } else {
5035             if ($force) {
5036                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5037                 kill 15, $pid;
5038             } else {
5039                 die "VM quit/powerdown failed\n";
5040             }
5041         }
5042
5043         # wait again
5044         $timeout = 10;
5045
5046         my $count = 0;
5047         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5048             $count++;
5049             sleep 1;
5050         }
5051
5052         if ($count >= $timeout) {
5053             warn "VM still running - terminating now with SIGKILL\n";
5054             kill 9, $pid;
5055             sleep 1;
5056         }
5057
5058         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5059    });
5060 }
5061
5062 sub vm_suspend {
5063     my ($vmid, $skiplock) = @_;
5064
5065     PVE::QemuConfig->lock_config($vmid, sub {
5066
5067         my $conf = PVE::QemuConfig->load_config($vmid);
5068
5069         PVE::QemuConfig->check_lock($conf)
5070             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5071
5072         vm_mon_cmd($vmid, "stop");
5073     });
5074 }
5075
5076 sub vm_resume {
5077     my ($vmid, $skiplock, $nocheck) = @_;
5078
5079     PVE::QemuConfig->lock_config($vmid, sub {
5080
5081         if (!$nocheck) {
5082
5083             my $conf = PVE::QemuConfig->load_config($vmid);
5084
5085             PVE::QemuConfig->check_lock($conf)
5086                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5087
5088             vm_mon_cmd($vmid, "cont");
5089
5090         } else {
5091             vm_mon_cmd_nocheck($vmid, "cont");
5092         }
5093     });
5094 }
5095
5096 sub vm_sendkey {
5097     my ($vmid, $skiplock, $key) = @_;
5098
5099     PVE::QemuConfig->lock_config($vmid, sub {
5100
5101         my $conf = PVE::QemuConfig->load_config($vmid);
5102
5103         # there is no qmp command, so we use the human monitor command
5104         vm_human_monitor_command($vmid, "sendkey $key");
5105     });
5106 }
5107
5108 sub vm_destroy {
5109     my ($storecfg, $vmid, $skiplock) = @_;
5110
5111     PVE::QemuConfig->lock_config($vmid, sub {
5112
5113         my $conf = PVE::QemuConfig->load_config($vmid);
5114
5115         if (!check_running($vmid)) {
5116             destroy_vm($storecfg, $vmid, undef, $skiplock);
5117         } else {
5118             die "VM $vmid is running - destroy failed\n";
5119         }
5120     });
5121 }
5122
5123 # pci helpers
5124
5125 sub file_write {
5126     my ($filename, $buf) = @_;
5127
5128     my $fh = IO::File->new($filename, "w");
5129     return undef if !$fh;
5130
5131     my $res = print $fh $buf;
5132
5133     $fh->close();
5134
5135     return $res;
5136 }
5137
5138 sub pci_device_info {
5139     my ($name) = @_;
5140
5141     my $res;
5142
5143     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5144     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5145
5146     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5147     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5148
5149     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5150     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5151
5152     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5153     return undef if !defined($product) || $product !~ s/^0x//;
5154
5155     $res = {
5156         name => $name,
5157         vendor => $vendor,
5158         product => $product,
5159         domain => $domain,
5160         bus => $bus,
5161         slot => $slot,
5162         func => $func,
5163         irq => $irq,
5164         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5165     };
5166
5167     return $res;
5168 }
5169
5170 sub pci_dev_reset {
5171     my ($dev) = @_;
5172
5173     my $name = $dev->{name};
5174
5175     my $fn = "$pcisysfs/devices/$name/reset";
5176
5177     return file_write($fn, "1");
5178 }
5179
5180 sub pci_dev_bind_to_vfio {
5181     my ($dev) = @_;
5182
5183     my $name = $dev->{name};
5184
5185     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5186
5187     if (!-d $vfio_basedir) {
5188         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5189     }
5190     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5191
5192     my $testdir = "$vfio_basedir/$name";
5193     return 1 if -d $testdir;
5194
5195     my $data = "$dev->{vendor} $dev->{product}";
5196     return undef if !file_write("$vfio_basedir/new_id", $data);
5197
5198     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5199     if (!file_write($fn, $name)) {
5200         return undef if -f $fn;
5201     }
5202
5203     $fn = "$vfio_basedir/bind";
5204     if (! -d $testdir) {
5205         return undef if !file_write($fn, $name);
5206     }
5207
5208     return -d $testdir;
5209 }
5210
5211 sub pci_dev_group_bind_to_vfio {
5212     my ($pciid) = @_;
5213
5214     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5215
5216     if (!-d $vfio_basedir) {
5217         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5218     }
5219     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5220
5221     # get IOMMU group devices
5222     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5223       my @devs = grep /^0000:/, readdir($D);
5224     closedir($D);
5225
5226     foreach my $pciid (@devs) {
5227         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5228
5229         # pci bridges, switches or root ports are not supported
5230         # they have a pci_bus subdirectory so skip them
5231         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5232
5233         my $info = pci_device_info($1);
5234         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5235     }
5236
5237     return 1;
5238 }
5239
5240 # vzdump restore implementaion
5241
5242 sub tar_archive_read_firstfile {
5243     my $archive = shift;
5244
5245     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5246
5247     # try to detect archive type first
5248     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5249         die "unable to open file '$archive'\n";
5250     my $firstfile = <$fh>;
5251     kill 15, $pid;
5252     close $fh;
5253
5254     die "ERROR: archive contaions no data\n" if !$firstfile;
5255     chomp $firstfile;
5256
5257     return $firstfile;
5258 }
5259
5260 sub tar_restore_cleanup {
5261     my ($storecfg, $statfile) = @_;
5262
5263     print STDERR "starting cleanup\n";
5264
5265     if (my $fd = IO::File->new($statfile, "r")) {
5266         while (defined(my $line = <$fd>)) {
5267             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5268                 my $volid = $2;
5269                 eval {
5270                     if ($volid =~ m|^/|) {
5271                         unlink $volid || die 'unlink failed\n';
5272                     } else {
5273                         PVE::Storage::vdisk_free($storecfg, $volid);
5274                     }
5275                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5276                 };
5277                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5278             } else {
5279                 print STDERR "unable to parse line in statfile - $line";
5280             }
5281         }
5282         $fd->close();
5283     }
5284 }
5285
5286 sub restore_archive {
5287     my ($archive, $vmid, $user, $opts) = @_;
5288
5289     my $format = $opts->{format};
5290     my $comp;
5291
5292     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5293         $format = 'tar' if !$format;
5294         $comp = 'gzip';
5295     } elsif ($archive =~ m/\.tar$/) {
5296         $format = 'tar' if !$format;
5297     } elsif ($archive =~ m/.tar.lzo$/) {
5298         $format = 'tar' if !$format;
5299         $comp = 'lzop';
5300     } elsif ($archive =~ m/\.vma$/) {
5301         $format = 'vma' if !$format;
5302     } elsif ($archive =~ m/\.vma\.gz$/) {
5303         $format = 'vma' if !$format;
5304         $comp = 'gzip';
5305     } elsif ($archive =~ m/\.vma\.lzo$/) {
5306         $format = 'vma' if !$format;
5307         $comp = 'lzop';
5308     } else {
5309         $format = 'vma' if !$format; # default
5310     }
5311
5312     # try to detect archive format
5313     if ($format eq 'tar') {
5314         return restore_tar_archive($archive, $vmid, $user, $opts);
5315     } else {
5316         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5317     }
5318 }
5319
5320 sub restore_update_config_line {
5321     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5322
5323     return if $line =~ m/^\#qmdump\#/;
5324     return if $line =~ m/^\#vzdump\#/;
5325     return if $line =~ m/^lock:/;
5326     return if $line =~ m/^unused\d+:/;
5327     return if $line =~ m/^parent:/;
5328     return if $line =~ m/^template:/; # restored VM is never a template
5329
5330     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5331     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5332         # try to convert old 1.X settings
5333         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5334         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5335             my ($model, $macaddr) = split(/\=/, $devconfig);
5336             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5337             my $net = {
5338                 model => $model,
5339                 bridge => "vmbr$ind",
5340                 macaddr => $macaddr,
5341             };
5342             my $netstr = print_net($net);
5343
5344             print $outfd "net$cookie->{netcount}: $netstr\n";
5345             $cookie->{netcount}++;
5346         }
5347     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5348         my ($id, $netstr) = ($1, $2);
5349         my $net = parse_net($netstr);
5350         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5351         $netstr = print_net($net);
5352         print $outfd "$id: $netstr\n";
5353     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5354         my $virtdev = $1;
5355         my $value = $3;
5356         my $di = parse_drive($virtdev, $value);
5357         if (defined($di->{backup}) && !$di->{backup}) {
5358             print $outfd "#$line";
5359         } elsif ($map->{$virtdev}) {
5360             delete $di->{format}; # format can change on restore
5361             $di->{file} = $map->{$virtdev};
5362             $value = print_drive($vmid, $di);
5363             print $outfd "$virtdev: $value\n";
5364         } else {
5365             print $outfd $line;
5366         }
5367     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5368         my ($uuid, $uuid_str);
5369         UUID::generate($uuid);
5370         UUID::unparse($uuid, $uuid_str);
5371         my $smbios1 = parse_smbios1($2);
5372         $smbios1->{uuid} = $uuid_str;
5373         print $outfd $1.print_smbios1($smbios1)."\n";
5374     } else {
5375         print $outfd $line;
5376     }
5377 }
5378
5379 sub scan_volids {
5380     my ($cfg, $vmid) = @_;
5381
5382     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5383
5384     my $volid_hash = {};
5385     foreach my $storeid (keys %$info) {
5386         foreach my $item (@{$info->{$storeid}}) {
5387             next if !($item->{volid} && $item->{size});
5388             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5389             $volid_hash->{$item->{volid}} = $item;
5390         }
5391     }
5392
5393     return $volid_hash;
5394 }
5395
5396 sub is_volume_in_use {
5397     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5398
5399     my $path = PVE::Storage::path($storecfg, $volid);
5400
5401     my $scan_config = sub {
5402         my ($cref, $snapname) = @_;
5403
5404         foreach my $key (keys %$cref) {
5405             my $value = $cref->{$key};
5406             if (is_valid_drivename($key)) {
5407                 next if $skip_drive && $key eq $skip_drive;
5408                 my $drive = parse_drive($key, $value);
5409                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5410                 return 1 if $volid eq $drive->{file};
5411                 if ($drive->{file} =~ m!^/!) {
5412                     return 1 if $drive->{file} eq $path;
5413                 } else {
5414                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5415                     next if !$storeid;
5416                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5417                     next if !$scfg;
5418                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5419                 }
5420             }
5421         }
5422
5423         return 0;
5424     };
5425
5426     return 1 if &$scan_config($conf);
5427
5428     undef $skip_drive;
5429
5430     foreach my $snapname (keys %{$conf->{snapshots}}) {
5431         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5432     }
5433
5434     return 0;
5435 }
5436
5437 sub update_disksize {
5438     my ($vmid, $conf, $volid_hash) = @_;
5439
5440     my $changes;
5441
5442     # used and unused disks
5443     my $referenced = {};
5444
5445     # Note: it is allowed to define multiple storages with same path (alias), so
5446     # we need to check both 'volid' and real 'path' (two different volid can point
5447     # to the same path).
5448
5449     my $referencedpath = {};
5450
5451     # update size info
5452     foreach my $opt (keys %$conf) {
5453         if (is_valid_drivename($opt)) {
5454             my $drive = parse_drive($opt, $conf->{$opt});
5455             my $volid = $drive->{file};
5456             next if !$volid;
5457
5458             $referenced->{$volid} = 1;
5459             if ($volid_hash->{$volid} &&
5460                 (my $path = $volid_hash->{$volid}->{path})) {
5461                 $referencedpath->{$path} = 1;
5462             }
5463
5464             next if drive_is_cdrom($drive);
5465             next if !$volid_hash->{$volid};
5466
5467             $drive->{size} = $volid_hash->{$volid}->{size};
5468             my $new = print_drive($vmid, $drive);
5469             if ($new ne $conf->{$opt}) {
5470                 $changes = 1;
5471                 $conf->{$opt} = $new;
5472             }
5473         }
5474     }
5475
5476     # remove 'unusedX' entry if volume is used
5477     foreach my $opt (keys %$conf) {
5478         next if $opt !~ m/^unused\d+$/;
5479         my $volid = $conf->{$opt};
5480         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5481         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5482             $changes = 1;
5483             delete $conf->{$opt};
5484         }
5485
5486         $referenced->{$volid} = 1;
5487         $referencedpath->{$path} = 1 if $path;
5488     }
5489
5490     foreach my $volid (sort keys %$volid_hash) {
5491         next if $volid =~ m/vm-$vmid-state-/;
5492         next if $referenced->{$volid};
5493         my $path = $volid_hash->{$volid}->{path};
5494         next if !$path; # just to be sure
5495         next if $referencedpath->{$path};
5496         $changes = 1;
5497         PVE::QemuConfig->add_unused_volume($conf, $volid);
5498         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5499     }
5500
5501     return $changes;
5502 }
5503
5504 sub rescan {
5505     my ($vmid, $nolock) = @_;
5506
5507     my $cfg = PVE::Storage::config();
5508
5509     my $volid_hash = scan_volids($cfg, $vmid);
5510
5511     my $updatefn =  sub {
5512         my ($vmid) = @_;
5513
5514         my $conf = PVE::QemuConfig->load_config($vmid);
5515
5516         PVE::QemuConfig->check_lock($conf);
5517
5518         my $vm_volids = {};
5519         foreach my $volid (keys %$volid_hash) {
5520             my $info = $volid_hash->{$volid};
5521             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5522         }
5523
5524         my $changes = update_disksize($vmid, $conf, $vm_volids);
5525
5526         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5527     };
5528
5529     if (defined($vmid)) {
5530         if ($nolock) {
5531             &$updatefn($vmid);
5532         } else {
5533             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5534         }
5535     } else {
5536         my $vmlist = config_list();
5537         foreach my $vmid (keys %$vmlist) {
5538             if ($nolock) {
5539                 &$updatefn($vmid);
5540             } else {
5541                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5542             }
5543         }
5544     }
5545 }
5546
5547 sub restore_vma_archive {
5548     my ($archive, $vmid, $user, $opts, $comp) = @_;
5549
5550     my $input = $archive eq '-' ? "<&STDIN" : undef;
5551     my $readfrom = $archive;
5552
5553     my $uncomp = '';
5554     if ($comp) {
5555         $readfrom = '-';
5556         my $qarchive = PVE::Tools::shellquote($archive);
5557         if ($comp eq 'gzip') {
5558             $uncomp = "zcat $qarchive|";
5559         } elsif ($comp eq 'lzop') {
5560             $uncomp = "lzop -d -c $qarchive|";
5561         } else {
5562             die "unknown compression method '$comp'\n";
5563         }
5564
5565     }
5566
5567     my $tmpdir = "/var/tmp/vzdumptmp$$";
5568     rmtree $tmpdir;
5569
5570     # disable interrupts (always do cleanups)
5571     local $SIG{INT} =
5572         local $SIG{TERM} =
5573         local $SIG{QUIT} =
5574         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5575
5576     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5577     POSIX::mkfifo($mapfifo, 0600);
5578     my $fifofh;
5579
5580     my $openfifo = sub {
5581         open($fifofh, '>', $mapfifo) || die $!;
5582     };
5583
5584     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5585
5586     my $oldtimeout;
5587     my $timeout = 5;
5588
5589     my $devinfo = {};
5590
5591     my $rpcenv = PVE::RPCEnvironment::get();
5592
5593     my $conffile = PVE::QemuConfig->config_file($vmid);
5594     my $tmpfn = "$conffile.$$.tmp";
5595
5596     # Note: $oldconf is undef if VM does not exists
5597     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5598     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5599
5600     my $print_devmap = sub {
5601         my $virtdev_hash = {};
5602
5603         my $cfgfn = "$tmpdir/qemu-server.conf";
5604
5605         # we can read the config - that is already extracted
5606         my $fh = IO::File->new($cfgfn, "r") ||
5607             "unable to read qemu-server.conf - $!\n";
5608
5609         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5610         if (-f $fwcfgfn) {
5611             my $pve_firewall_dir = '/etc/pve/firewall';
5612             mkdir $pve_firewall_dir; # make sure the dir exists
5613             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5614         }
5615
5616         while (defined(my $line = <$fh>)) {
5617             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5618                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5619                 die "archive does not contain data for drive '$virtdev'\n"
5620                     if !$devinfo->{$devname};
5621                 if (defined($opts->{storage})) {
5622                     $storeid = $opts->{storage} || 'local';
5623                 } elsif (!$storeid) {
5624                     $storeid = 'local';
5625                 }
5626                 $format = 'raw' if !$format;
5627                 $devinfo->{$devname}->{devname} = $devname;
5628                 $devinfo->{$devname}->{virtdev} = $virtdev;
5629                 $devinfo->{$devname}->{format} = $format;
5630                 $devinfo->{$devname}->{storeid} = $storeid;
5631
5632                 # check permission on storage
5633                 my $pool = $opts->{pool}; # todo: do we need that?
5634                 if ($user ne 'root@pam') {
5635                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5636                 }
5637
5638                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5639             }
5640         }
5641
5642         foreach my $devname (keys %$devinfo) {
5643             die "found no device mapping information for device '$devname'\n"
5644                 if !$devinfo->{$devname}->{virtdev};
5645         }
5646
5647         my $cfg = PVE::Storage::config();
5648
5649         # create empty/temp config
5650         if ($oldconf) {
5651             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5652             foreach_drive($oldconf, sub {
5653                 my ($ds, $drive) = @_;
5654
5655                 return if drive_is_cdrom($drive);
5656
5657                 my $volid = $drive->{file};
5658
5659                 return if !$volid || $volid =~ m|^/|;
5660
5661                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5662                 return if !$path || !$owner || ($owner != $vmid);
5663
5664                 # Note: only delete disk we want to restore
5665                 # other volumes will become unused
5666                 if ($virtdev_hash->{$ds}) {
5667                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5668                     if (my $err = $@) {
5669                         warn $err;
5670                     }
5671                 }
5672             });
5673
5674             # delete vmstate files
5675             # since after the restore we have no snapshots anymore
5676             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5677                 my $snap = $oldconf->{snapshots}->{$snapname};
5678                 if ($snap->{vmstate}) {
5679                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5680                     if (my $err = $@) {
5681                         warn $err;
5682                     }
5683                 }
5684             }
5685         }
5686
5687         my $map = {};
5688         foreach my $virtdev (sort keys %$virtdev_hash) {
5689             my $d = $virtdev_hash->{$virtdev};
5690             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5691             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5692
5693             # test if requested format is supported
5694             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5695             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5696             $d->{format} = $defFormat if !$supported;
5697
5698             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5699                                                   $d->{format}, undef, $alloc_size);
5700             print STDERR "new volume ID is '$volid'\n";
5701             $d->{volid} = $volid;
5702             my $path = PVE::Storage::path($cfg, $volid);
5703
5704             PVE::Storage::activate_volumes($cfg,[$volid]);
5705
5706             my $write_zeros = 1;
5707             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5708                 $write_zeros = 0;
5709             }
5710
5711             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5712
5713             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5714             $map->{$virtdev} = $volid;
5715         }
5716
5717         $fh->seek(0, 0) || die "seek failed - $!\n";
5718
5719         my $outfd = new IO::File ($tmpfn, "w") ||
5720             die "unable to write config for VM $vmid\n";
5721
5722         my $cookie = { netcount => 0 };
5723         while (defined(my $line = <$fh>)) {
5724             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5725         }
5726
5727         $fh->close();
5728         $outfd->close();
5729     };
5730
5731     eval {
5732         # enable interrupts
5733         local $SIG{INT} =
5734             local $SIG{TERM} =
5735             local $SIG{QUIT} =
5736             local $SIG{HUP} =
5737             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5738         local $SIG{ALRM} = sub { die "got timeout\n"; };
5739
5740         $oldtimeout = alarm($timeout);
5741
5742         my $parser = sub {
5743             my $line = shift;
5744
5745             print "$line\n";
5746
5747             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5748                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5749                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5750             } elsif ($line =~ m/^CTIME: /) {
5751                 # we correctly received the vma config, so we can disable
5752                 # the timeout now for disk allocation (set to 10 minutes, so
5753                 # that we always timeout if something goes wrong)
5754                 alarm(600);
5755                 &$print_devmap();
5756                 print $fifofh "done\n";
5757                 my $tmp = $oldtimeout || 0;
5758                 $oldtimeout = undef;
5759                 alarm($tmp);
5760                 close($fifofh);
5761             }
5762         };
5763
5764         print "restore vma archive: $cmd\n";
5765         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5766     };
5767     my $err = $@;
5768
5769     alarm($oldtimeout) if $oldtimeout;
5770
5771     my $vollist = [];
5772     foreach my $devname (keys %$devinfo) {
5773         my $volid = $devinfo->{$devname}->{volid};
5774         push @$vollist, $volid if $volid;
5775     }
5776
5777     my $cfg = PVE::Storage::config();
5778     PVE::Storage::deactivate_volumes($cfg, $vollist);
5779
5780     unlink $mapfifo;
5781
5782     if ($err) {
5783         rmtree $tmpdir;
5784         unlink $tmpfn;
5785
5786         foreach my $devname (keys %$devinfo) {
5787             my $volid = $devinfo->{$devname}->{volid};
5788             next if !$volid;
5789             eval {
5790                 if ($volid =~ m|^/|) {
5791                     unlink $volid || die 'unlink failed\n';
5792                 } else {
5793                     PVE::Storage::vdisk_free($cfg, $volid);
5794                 }
5795                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5796             };
5797             print STDERR "unable to cleanup '$volid' - $@" if $@;
5798         }
5799         die $err;
5800     }
5801
5802     rmtree $tmpdir;
5803
5804     rename($tmpfn, $conffile) ||
5805         die "unable to commit configuration file '$conffile'\n";
5806
5807     PVE::Cluster::cfs_update(); # make sure we read new file
5808
5809     eval { rescan($vmid, 1); };
5810     warn $@ if $@;
5811 }
5812
5813 sub restore_tar_archive {
5814     my ($archive, $vmid, $user, $opts) = @_;
5815
5816     if ($archive ne '-') {
5817         my $firstfile = tar_archive_read_firstfile($archive);
5818         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5819             if $firstfile ne 'qemu-server.conf';
5820     }
5821
5822     my $storecfg = PVE::Storage::config();
5823
5824     # destroy existing data - keep empty config
5825     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5826     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5827
5828     my $tocmd = "/usr/lib/qemu-server/qmextract";
5829
5830     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5831     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5832     $tocmd .= ' --prealloc' if $opts->{prealloc};
5833     $tocmd .= ' --info' if $opts->{info};
5834
5835     # tar option "xf" does not autodetect compression when read from STDIN,
5836     # so we pipe to zcat
5837     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5838         PVE::Tools::shellquote("--to-command=$tocmd");
5839
5840     my $tmpdir = "/var/tmp/vzdumptmp$$";
5841     mkpath $tmpdir;
5842
5843     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5844     local $ENV{VZDUMP_VMID} = $vmid;
5845     local $ENV{VZDUMP_USER} = $user;
5846
5847     my $conffile = PVE::QemuConfig->config_file($vmid);
5848     my $tmpfn = "$conffile.$$.tmp";
5849
5850     # disable interrupts (always do cleanups)
5851     local $SIG{INT} =
5852         local $SIG{TERM} =
5853         local $SIG{QUIT} =
5854         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5855
5856     eval {
5857         # enable interrupts
5858         local $SIG{INT} =
5859             local $SIG{TERM} =
5860             local $SIG{QUIT} =
5861             local $SIG{HUP} =
5862             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5863
5864         if ($archive eq '-') {
5865             print "extracting archive from STDIN\n";
5866             run_command($cmd, input => "<&STDIN");
5867         } else {
5868             print "extracting archive '$archive'\n";
5869             run_command($cmd);
5870         }
5871
5872         return if $opts->{info};
5873
5874         # read new mapping
5875         my $map = {};
5876         my $statfile = "$tmpdir/qmrestore.stat";
5877         if (my $fd = IO::File->new($statfile, "r")) {
5878             while (defined (my $line = <$fd>)) {
5879                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5880                     $map->{$1} = $2 if $1;
5881                 } else {
5882                     print STDERR "unable to parse line in statfile - $line\n";
5883                 }
5884             }
5885             $fd->close();
5886         }
5887
5888         my $confsrc = "$tmpdir/qemu-server.conf";
5889
5890         my $srcfd = new IO::File($confsrc, "r") ||
5891             die "unable to open file '$confsrc'\n";
5892
5893         my $outfd = new IO::File ($tmpfn, "w") ||
5894             die "unable to write config for VM $vmid\n";
5895
5896         my $cookie = { netcount => 0 };
5897         while (defined (my $line = <$srcfd>)) {
5898             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5899         }
5900
5901         $srcfd->close();
5902         $outfd->close();
5903     };
5904     my $err = $@;
5905
5906     if ($err) {
5907
5908         unlink $tmpfn;
5909
5910         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5911
5912         die $err;
5913     }
5914
5915     rmtree $tmpdir;
5916
5917     rename $tmpfn, $conffile ||
5918         die "unable to commit configuration file '$conffile'\n";
5919
5920     PVE::Cluster::cfs_update(); # make sure we read new file
5921
5922     eval { rescan($vmid, 1); };
5923     warn $@ if $@;
5924 };
5925
5926 sub foreach_storage_used_by_vm {
5927     my ($conf, $func) = @_;
5928
5929     my $sidhash = {};
5930
5931     foreach_drive($conf, sub {
5932         my ($ds, $drive) = @_;
5933         return if drive_is_cdrom($drive);
5934
5935         my $volid = $drive->{file};
5936
5937         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5938         $sidhash->{$sid} = $sid if $sid;
5939     });
5940
5941     foreach my $sid (sort keys %$sidhash) {
5942         &$func($sid);
5943     }
5944 }
5945
5946 sub do_snapshots_with_qemu {
5947     my ($storecfg, $volid) = @_;
5948
5949     my $storage_name = PVE::Storage::parse_volume_id($volid);
5950
5951     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
5952         && !$storecfg->{ids}->{$storage_name}->{krbd}){
5953         return 1;
5954     }
5955
5956     if ($volid =~ m/\.(qcow2|qed)$/){
5957         return 1;
5958     }
5959
5960     return undef;
5961 }
5962
5963 sub qga_check_running {
5964     my ($vmid) = @_;
5965
5966     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
5967     if ($@) {
5968         warn "Qemu Guest Agent is not running - $@";
5969         return 0;
5970     }
5971     return 1;
5972 }
5973
5974 sub template_create {
5975     my ($vmid, $conf, $disk) = @_;
5976
5977     my $storecfg = PVE::Storage::config();
5978
5979     foreach_drive($conf, sub {
5980         my ($ds, $drive) = @_;
5981
5982         return if drive_is_cdrom($drive);
5983         return if $disk && $ds ne $disk;
5984
5985         my $volid = $drive->{file};
5986         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
5987
5988         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
5989         $drive->{file} = $voliddst;
5990         $conf->{$ds} = print_drive($vmid, $drive);
5991         PVE::QemuConfig->write_config($vmid, $conf);
5992     });
5993 }
5994
5995 sub qemu_img_convert {
5996     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
5997
5998     my $storecfg = PVE::Storage::config();
5999     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6000     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6001
6002     if ($src_storeid && $dst_storeid) {
6003
6004         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6005
6006         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6007         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6008
6009         my $src_format = qemu_img_format($src_scfg, $src_volname);
6010         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6011
6012         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6013         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6014
6015         my $cmd = [];
6016         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6017         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6018         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6019         if ($is_zero_initialized) {
6020             push @$cmd, "zeroinit:$dst_path";
6021         } else {
6022             push @$cmd, $dst_path;
6023         }
6024
6025         my $parser = sub {
6026             my $line = shift;
6027             if($line =~ m/\((\S+)\/100\%\)/){
6028                 my $percent = $1;
6029                 my $transferred = int($size * $percent / 100);
6030                 my $remaining = $size - $transferred;
6031
6032                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6033             }
6034
6035         };
6036
6037         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6038         my $err = $@;
6039         die "copy failed: $err" if $err;
6040     }
6041 }
6042
6043 sub qemu_img_format {
6044     my ($scfg, $volname) = @_;
6045
6046     if ($scfg->{path} && $volname =~ m/\.(raw|cow|qcow|qcow2|qed|vmdk|cloop)$/) {
6047         return $1;
6048     } else {
6049         return "raw";
6050     }
6051 }
6052
6053 sub qemu_drive_mirror {
6054     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6055
6056     $jobs = {} if !$jobs;
6057
6058     my $qemu_target;
6059     my $format;
6060     $jobs->{"drive-$drive"} = {};
6061
6062     if ($dst_volid =~ /^nbd:/) {
6063         $qemu_target = $dst_volid;
6064         $format = "nbd";
6065     } else {
6066         my $storecfg = PVE::Storage::config();
6067         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6068
6069         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6070
6071         $format = qemu_img_format($dst_scfg, $dst_volname);
6072
6073         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6074
6075         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6076     }
6077
6078     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6079     $opts->{format} = $format if $format;
6080
6081     print "drive mirror is starting for drive-$drive\n";
6082
6083     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6084
6085     if (my $err = $@) {
6086         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6087         die "mirroring error: $err";
6088     }
6089
6090     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6091 }
6092
6093 sub qemu_drive_mirror_monitor {
6094     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6095
6096     eval {
6097         my $err_complete = 0;
6098
6099         while (1) {
6100             die "storage migration timed out\n" if $err_complete > 300;
6101
6102             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6103
6104             my $running_mirror_jobs = {};
6105             foreach my $stat (@$stats) {
6106                 next if $stat->{type} ne 'mirror';
6107                 $running_mirror_jobs->{$stat->{device}} = $stat;
6108             }
6109
6110             my $readycounter = 0;
6111
6112             foreach my $job (keys %$jobs) {
6113
6114                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6115                     print "$job : finished\n";
6116                     delete $jobs->{$job};
6117                     next;
6118                 }
6119
6120                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6121
6122                 my $busy = $running_mirror_jobs->{$job}->{busy};
6123                 my $ready = $running_mirror_jobs->{$job}->{ready};
6124                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6125                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6126                     my $remaining = $total - $transferred;
6127                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6128
6129                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6130                 }
6131
6132                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6133             }
6134
6135             last if scalar(keys %$jobs) == 0;
6136
6137             if ($readycounter == scalar(keys %$jobs)) {
6138                 print "all mirroring jobs are ready \n";
6139                 last if $skipcomplete; #do the complete later
6140
6141                 if ($vmiddst && $vmiddst != $vmid) {
6142                     my $agent_running = $qga && qga_check_running($vmid);
6143                     if ($agent_running) {
6144                         print "freeze filesystem\n";
6145                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6146                     } else {
6147                         print "suspend vm\n";
6148                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6149                     }
6150
6151                     # if we clone a disk for a new target vm, we don't switch the disk
6152                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6153
6154                     if ($agent_running) {
6155                         print "unfreeze filesystem\n";
6156                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6157                     } else {
6158                         print "resume vm\n";
6159                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6160                     }
6161
6162                     last;
6163                 } else {
6164
6165                     foreach my $job (keys %$jobs) {
6166                         # try to switch the disk if source and destination are on the same guest
6167                         print "$job: Completing block job...\n";
6168
6169                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6170                         if ($@ =~ m/cannot be completed/) {
6171                             print "$job: Block job cannot be completed, try again.\n";
6172                             $err_complete++;
6173                         }else {
6174                             print "$job: Completed successfully.\n";
6175                             $jobs->{$job}->{complete} = 1;
6176                         }
6177                     }
6178                 }
6179             }
6180             sleep 1;
6181         }
6182     };
6183     my $err = $@;
6184
6185     if ($err) {
6186         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6187         die "mirroring error: $err";
6188     }
6189
6190 }
6191
6192 sub qemu_blockjobs_cancel {
6193     my ($vmid, $jobs) = @_;
6194
6195     foreach my $job (keys %$jobs) {
6196         print "$job: Cancelling block job\n";
6197         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6198         $jobs->{$job}->{cancel} = 1;
6199     }
6200
6201     while (1) {
6202         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6203
6204         my $running_jobs = {};
6205         foreach my $stat (@$stats) {
6206             $running_jobs->{$stat->{device}} = $stat;
6207         }
6208
6209         foreach my $job (keys %$jobs) {
6210
6211             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6212                 print "$job: Done.\n";
6213                 delete $jobs->{$job};
6214             }
6215         }
6216
6217         last if scalar(keys %$jobs) == 0;
6218
6219         sleep 1;
6220     }
6221 }
6222
6223 sub clone_disk {
6224     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6225         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6226
6227     my $newvolid;
6228
6229     if (!$full) {
6230         print "create linked clone of drive $drivename ($drive->{file})\n";
6231         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6232         push @$newvollist, $newvolid;
6233     } else {
6234
6235         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6236         $storeid = $storage if $storage;
6237
6238         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6239         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6240
6241         print "create full clone of drive $drivename ($drive->{file})\n";
6242         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, undef, ($size/1024));
6243         push @$newvollist, $newvolid;
6244
6245         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6246
6247         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6248         if (!$running || $snapname) {
6249             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6250         } else {
6251
6252             my $kvmver = get_running_qemu_version ($vmid);
6253             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6254                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6255                     if $drive->{iothread};
6256             }
6257
6258             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6259         }
6260     }
6261
6262     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6263
6264     my $disk = $drive;
6265     $disk->{format} = undef;
6266     $disk->{file} = $newvolid;
6267     $disk->{size} = $size;
6268
6269     return $disk;
6270 }
6271
6272 # this only works if VM is running
6273 sub get_current_qemu_machine {
6274     my ($vmid) = @_;
6275
6276     my $cmd = { execute => 'query-machines', arguments => {} };
6277     my $res = vm_qmp_command($vmid, $cmd);
6278
6279     my ($current, $default);
6280     foreach my $e (@$res) {
6281         $default = $e->{name} if $e->{'is-default'};
6282         $current = $e->{name} if $e->{'is-current'};
6283     }
6284
6285     # fallback to the default machine if current is not supported by qemu
6286     return $current || $default || 'pc';
6287 }
6288
6289 sub get_running_qemu_version {
6290     my ($vmid) = @_;
6291     my $cmd = { execute => 'query-version', arguments => {} };
6292     my $res = vm_qmp_command($vmid, $cmd);
6293     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6294 }
6295
6296 sub qemu_machine_feature_enabled {
6297     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6298
6299     my $current_major;
6300     my $current_minor;
6301
6302     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6303
6304         $current_major = $3;
6305         $current_minor = $4;
6306
6307     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6308
6309         $current_major = $1;
6310         $current_minor = $2;
6311     }
6312
6313     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6314
6315
6316 }
6317
6318 sub qemu_machine_pxe {
6319     my ($vmid, $conf, $machine) = @_;
6320
6321     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6322
6323     foreach my $opt (keys %$conf) {
6324         next if $opt !~ m/^net(\d+)$/;
6325         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6326         next if !$net;
6327         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6328         return $machine.".pxe" if $romfile =~ m/pxe/;
6329         last;
6330     }
6331
6332     return $machine;
6333 }
6334
6335 sub qemu_use_old_bios_files {
6336     my ($machine_type) = @_;
6337
6338     return if !$machine_type;
6339
6340     my $use_old_bios_files = undef;
6341
6342     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6343         $machine_type = $1;
6344         $use_old_bios_files = 1;
6345     } else {
6346         my $kvmver = kvm_user_version();
6347         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6348         # load new efi bios files on migration. So this hack is required to allow
6349         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6350         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6351         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6352     }
6353
6354     return ($use_old_bios_files, $machine_type);
6355 }
6356
6357 sub create_efidisk {
6358     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6359
6360     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6361
6362     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6363     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6364     PVE::Storage::activate_volumes($storecfg, [$volid]);
6365
6366     my $path = PVE::Storage::path($storecfg, $volid);
6367     eval {
6368         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6369     };
6370     die "Copying EFI vars image failed: $@" if $@;
6371
6372     return ($volid, $vars_size);
6373 }
6374
6375 sub lspci {
6376
6377     my $devices = {};
6378
6379     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6380             my (undef, $id, $function) = @_;
6381             my $res = { id => $id, function => $function};
6382             push @{$devices->{$id}}, $res;
6383     });
6384
6385     # Entries should be sorted by functions.
6386     foreach my $id (keys %$devices) {
6387         my $dev = $devices->{$id};
6388         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6389     }
6390
6391     return $devices;
6392 }
6393
6394 sub vm_iothreads_list {
6395     my ($vmid) = @_;
6396
6397     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6398
6399     my $iothreads = {};
6400     foreach my $iothread (@$res) {
6401         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6402     }
6403
6404     return $iothreads;
6405 }
6406
6407 sub scsihw_infos {
6408     my ($conf, $drive) = @_;
6409
6410     my $maxdev = 0;
6411
6412     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6413         $maxdev = 7;
6414     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6415         $maxdev = 1;
6416     } else {
6417         $maxdev = 256;
6418     }
6419
6420     my $controller = int($drive->{index} / $maxdev);
6421     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6422
6423     return ($maxdev, $controller, $controller_prefix);
6424 }
6425
6426 sub add_hyperv_enlightenments {
6427     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6428
6429     return if $winversion < 6;
6430     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6431
6432     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6433
6434     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6435         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6436         push @$cpuFlags , 'hv_vapic';
6437         push @$cpuFlags , 'hv_time';
6438     } else {
6439         push @$cpuFlags , 'hv_spinlocks=0xffff';
6440     }
6441
6442     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6443         push @$cpuFlags , 'hv_reset';
6444         push @$cpuFlags , 'hv_vpindex';
6445         push @$cpuFlags , 'hv_runtime';
6446     }
6447
6448     if ($winversion >= 7) {
6449         push @$cpuFlags , 'hv_relaxed';
6450     }
6451 }
6452
6453 sub windows_version {
6454     my ($ostype) = @_;
6455
6456     return 0 if !$ostype;
6457
6458     my $winversion = 0;
6459
6460     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6461         $winversion = 5;
6462     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6463         $winversion = 6;
6464     } elsif ($ostype =~ m/^win(\d+)$/) {
6465         $winversion = $1;
6466     }
6467
6468     return $winversion;
6469 }
6470
6471 sub resolve_dst_disk_format {
6472         my ($storecfg, $storeid, $src_volname, $format) = @_;
6473         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6474
6475         if (!$format) {
6476             # if no target format is specified, use the source disk format as hint
6477             if ($src_volname) {
6478                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6479                 $format = qemu_img_format($scfg, $src_volname);
6480             } else {
6481                 return $defFormat;
6482             }
6483         }
6484
6485         # test if requested format is supported - else use default
6486         my $supported = grep { $_ eq $format } @$validFormats;
6487         $format = $defFormat if !$supported;
6488         return $format;
6489 }
6490
6491 sub resolve_first_disk {
6492     my $conf = shift;
6493     my @disks = PVE::QemuServer::valid_drive_names();
6494     my $firstdisk;
6495     foreach my $ds (reverse @disks) {
6496         next if !$conf->{$ds};
6497         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6498         next if PVE::QemuServer::drive_is_cdrom($disk);
6499         $firstdisk = $ds;
6500     }
6501     return $firstdisk;
6502 }
6503
6504 sub generate_smbios1_uuid {
6505     my ($uuid, $uuid_str);
6506     UUID::generate($uuid);
6507     UUID::unparse($uuid, $uuid_str);
6508     return "uuid=$uuid_str";
6509 }
6510
6511 # bash completion helper
6512
6513 sub complete_backup_archives {
6514     my ($cmdname, $pname, $cvalue) = @_;
6515
6516     my $cfg = PVE::Storage::config();
6517
6518     my $storeid;
6519
6520     if ($cvalue =~ m/^([^:]+):/) {
6521         $storeid = $1;
6522     }
6523
6524     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6525
6526     my $res = [];
6527     foreach my $id (keys %$data) {
6528         foreach my $item (@{$data->{$id}}) {
6529             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6530             push @$res, $item->{volid} if defined($item->{volid});
6531         }
6532     }
6533
6534     return $res;
6535 }
6536
6537 my $complete_vmid_full = sub {
6538     my ($running) = @_;
6539
6540     my $idlist = vmstatus();
6541
6542     my $res = [];
6543
6544     foreach my $id (keys %$idlist) {
6545         my $d = $idlist->{$id};
6546         if (defined($running)) {
6547             next if $d->{template};
6548             next if $running && $d->{status} ne 'running';
6549             next if !$running && $d->{status} eq 'running';
6550         }
6551         push @$res, $id;
6552
6553     }
6554     return $res;
6555 };
6556
6557 sub complete_vmid {
6558     return &$complete_vmid_full();
6559 }
6560
6561 sub complete_vmid_stopped {
6562     return &$complete_vmid_full(0);
6563 }
6564
6565 sub complete_vmid_running {
6566     return &$complete_vmid_full(1);
6567 }
6568
6569 sub complete_storage {
6570
6571     my $cfg = PVE::Storage::config();
6572     my $ids = $cfg->{ids};
6573
6574     my $res = [];
6575     foreach my $sid (keys %$ids) {
6576         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6577         next if !$ids->{$sid}->{content}->{images};
6578         push @$res, $sid;
6579     }
6580
6581     return $res;
6582 }
6583
6584 sub nbd_stop {
6585     my ($vmid) = @_;
6586
6587     vm_mon_cmd($vmid, 'nbd-server-stop');
6588 }
6589
6590 1;