PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use Time::HiRes qw(gettimeofday);
  38 use File::Copy qw(copy);
  39 use URI::Escape;
  40
  41 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  42 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  49
  50 # Note about locking: we use flock on the config file protect
  51 # against concurent actions.
  52 # Aditionaly, we have a 'lock' setting in the config file. This
  53 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  54 # allowed when such lock is set. But you can ignore this kind of
  55 # lock with the --skiplock flag.
  56
  57 cfs_register_file('/qemu-server/',
  58                   \&parse_vm_config,
  59                   \&write_vm_config);
  60
  61 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  62     description => "Some command save/restore state from this location.",
  63     type => 'string',
  64     maxLength => 128,
  65     optional => 1,
  66 });
  67
  68 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  69     description => "The name of the snapshot.",
  70     type => 'string', format => 'pve-configid',
  71     maxLength => 40,
  72 });
  73
  74 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  75     type => 'string',
  76     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  77     description => "The drive's backing file's data format.",
  78     optional => 1,
  79 });
  80
  81 #no warnings 'redefine';
  82
  83 sub cgroups_write {
  84    my ($controller, $vmid, $option, $value) = @_;
  85
  86    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  87    PVE::ProcFSTools::write_proc_entry($path, $value);
  88
  89 }
  90
  91 my $nodename = PVE::INotify::nodename();
  92
  93 mkdir "/etc/pve/nodes/$nodename";
  94 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  95 mkdir $confdir;
  96
  97 my $var_run_tmpdir = "/var/run/qemu-server";
  98 mkdir $var_run_tmpdir;
  99
 100 my $lock_dir = "/var/lock/qemu-server";
 101 mkdir $lock_dir;
 102
 103 my $pcisysfs = "/sys/bus/pci";
 104
 105 my $cpu_vendor_list = {
 106     # Intel CPUs
 107     486 => 'GenuineIntel',
 108     pentium => 'GenuineIntel',
 109     pentium2  => 'GenuineIntel',
 110     pentium3  => 'GenuineIntel',
 111     coreduo => 'GenuineIntel',
 112     core2duo => 'GenuineIntel',
 113     Conroe  => 'GenuineIntel',
 114     Penryn  => 'GenuineIntel',
 115     Nehalem  => 'GenuineIntel',
 116     'Nehalem-IBRS'  => 'GenuineIntel',
 117     Westmere => 'GenuineIntel',
 118     'Westmere-IBRS' => 'GenuineIntel',
 119     SandyBridge => 'GenuineIntel',
 120     'SandyBridge-IBRS' => 'GenuineIntel',
 121     IvyBridge => 'GenuineIntel',
 122     'IvyBridge-IBRS' => 'GenuineIntel',
 123     Haswell => 'GenuineIntel',
 124     'Haswell-IBRS' => 'GenuineIntel',
 125     'Haswell-noTSX' => 'GenuineIntel',
 126     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 127     Broadwell => 'GenuineIntel',
 128     'Broadwell-IBRS' => 'GenuineIntel',
 129     'Broadwell-noTSX' => 'GenuineIntel',
 130     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 131     'Skylake-Client' => 'GenuineIntel',
 132     'Skylake-Client-IBRS' => 'GenuineIntel',
 133     'Skylake-Server' => 'GenuineIntel',
 134     'Skylake-Server-IBRS' => 'GenuineIntel',
 135
 136     # AMD CPUs
 137     athlon => 'AuthenticAMD',
 138     phenom  => 'AuthenticAMD',
 139     Opteron_G1  => 'AuthenticAMD',
 140     Opteron_G2  => 'AuthenticAMD',
 141     Opteron_G3  => 'AuthenticAMD',
 142     Opteron_G4  => 'AuthenticAMD',
 143     Opteron_G5  => 'AuthenticAMD',
 144     EPYC => 'AuthenticAMD',
 145     'EPYC-IBPB' => 'AuthenticAMD',
 146
 147     # generic types, use vendor from host node
 148     host => 'default',
 149     kvm32 => 'default',
 150     kvm64 => 'default',
 151     qemu32 => 'default',
 152     qemu64 => 'default',
 153     max => 'default',
 154 };
 155
 156 my $cpu_flag = qr/[+-](pcid|spec-ctrl)/;
 157
 158 my $cpu_fmt = {
 159     cputype => {
 160         description => "Emulated CPU type.",
 161         type => 'string',
 162         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 163         default => 'kvm64',
 164         default_key => 1,
 165     },
 166     hidden => {
 167         description => "Do not identify as a KVM virtual machine.",
 168         type => 'boolean',
 169         optional => 1,
 170         default => 0
 171     },
 172     flags => {
 173         description => "List of additional CPU flags separated by ';'."
 174                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 175                      . " Currently supported flags: 'pcid', 'spec-ctrl'.",
 176         format_description => '+FLAG[;-FLAG...]',
 177         type => 'string',
 178         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 179         optional => 1,
 180     },
 181 };
 182
 183 my $watchdog_fmt = {
 184     model => {
 185         default_key => 1,
 186         type => 'string',
 187         enum => [qw(i6300esb ib700)],
 188         description => "Watchdog type to emulate.",
 189         default => 'i6300esb',
 190         optional => 1,
 191     },
 192     action => {
 193         type => 'string',
 194         enum => [qw(reset shutdown poweroff pause debug none)],
 195         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 196         optional => 1,
 197     },
 198 };
 199 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 200
 201 my $confdesc = {
 202     onboot => {
 203         optional => 1,
 204         type => 'boolean',
 205         description => "Specifies whether a VM will be started during system bootup.",
 206         default => 0,
 207     },
 208     autostart => {
 209         optional => 1,
 210         type => 'boolean',
 211         description => "Automatic restart after crash (currently ignored).",
 212         default => 0,
 213     },
 214     hotplug => {
 215         optional => 1,
 216         type => 'string', format => 'pve-hotplug-features',
 217         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 218         default => 'network,disk,usb',
 219     },
 220     reboot => {
 221         optional => 1,
 222         type => 'boolean',
 223         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 224         default => 1,
 225     },
 226     lock => {
 227         optional => 1,
 228         type => 'string',
 229         description => "Lock/unlock the VM.",
 230         enum => [qw(migrate backup snapshot rollback)],
 231     },
 232     cpulimit => {
 233         optional => 1,
 234         type => 'number',
 235         description => "Limit of CPU usage.",
 236         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 237         minimum => 0,
 238         maximum => 128,
 239         default => 0,
 240     },
 241     cpuunits => {
 242         optional => 1,
 243         type => 'integer',
 244         description => "CPU weight for a VM.",
 245         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 246         minimum => 2,
 247         maximum => 262144,
 248         default => 1024,
 249     },
 250     memory => {
 251         optional => 1,
 252         type => 'integer',
 253         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 254         minimum => 16,
 255         default => 512,
 256     },
 257     balloon => {
 258         optional => 1,
 259         type => 'integer',
 260         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 261         minimum => 0,
 262     },
 263     shares => {
 264         optional => 1,
 265         type => 'integer',
 266         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 267         minimum => 0,
 268         maximum => 50000,
 269         default => 1000,
 270     },
 271     keyboard => {
 272         optional => 1,
 273         type => 'string',
 274         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.".
 275                        "It should not be necessary to set it.",
 276         enum => PVE::Tools::kvmkeymaplist(),
 277         default => undef,
 278     },
 279     name => {
 280         optional => 1,
 281         type => 'string', format => 'dns-name',
 282         description => "Set a name for the VM. Only used on the configuration web interface.",
 283     },
 284     scsihw => {
 285         optional => 1,
 286         type => 'string',
 287         description => "SCSI controller model",
 288         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 289         default => 'lsi',
 290     },
 291     description => {
 292         optional => 1,
 293         type => 'string',
 294         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 295     },
 296     ostype => {
 297         optional => 1,
 298         type => 'string',
 299         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 300         description => "Specify guest operating system.",
 301         verbose_description => <<EODESC,
 302 Specify guest operating system. This is used to enable special
 303 optimization/features for specific operating systems:
 304
 305 [horizontal]
 306 other;; unspecified OS
 307 wxp;; Microsoft Windows XP
 308 w2k;; Microsoft Windows 2000
 309 w2k3;; Microsoft Windows 2003
 310 w2k8;; Microsoft Windows 2008
 311 wvista;; Microsoft Windows Vista
 312 win7;; Microsoft Windows 7
 313 win8;; Microsoft Windows 8/2012/2012r2
 314 win10;; Microsoft Windows 10/2016
 315 l24;; Linux 2.4 Kernel
 316 l26;; Linux 2.6/3.X Kernel
 317 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 318 EODESC
 319     },
 320     boot => {
 321         optional => 1,
 322         type => 'string',
 323         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 324         pattern => '[acdn]{1,4}',
 325         default => 'cdn',
 326     },
 327     bootdisk => {
 328         optional => 1,
 329         type => 'string', format => 'pve-qm-bootdisk',
 330         description => "Enable booting from specified disk.",
 331         pattern => '(ide|sata|scsi|virtio)\d+',
 332     },
 333     smp => {
 334         optional => 1,
 335         type => 'integer',
 336         description => "The number of CPUs. Please use option -sockets instead.",
 337         minimum => 1,
 338         default => 1,
 339     },
 340     sockets => {
 341         optional => 1,
 342         type => 'integer',
 343         description => "The number of CPU sockets.",
 344         minimum => 1,
 345         default => 1,
 346     },
 347     cores => {
 348         optional => 1,
 349         type => 'integer',
 350         description => "The number of cores per socket.",
 351         minimum => 1,
 352         default => 1,
 353     },
 354     numa => {
 355         optional => 1,
 356         type => 'boolean',
 357         description => "Enable/disable NUMA.",
 358         default => 0,
 359     },
 360     hugepages => {
 361         optional => 1,
 362         type => 'string',
 363         description => "Enable/disable hugepages memory.",
 364         enum => [qw(any 2 1024)],
 365     },
 366     vcpus => {
 367         optional => 1,
 368         type => 'integer',
 369         description => "Number of hotplugged vcpus.",
 370         minimum => 1,
 371         default => 0,
 372     },
 373     acpi => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Enable/disable ACPI.",
 377         default => 1,
 378     },
 379     agent => {
 380         optional => 1,
 381         type => 'boolean',
 382         description => "Enable/disable Qemu GuestAgent.",
 383         default => 0,
 384     },
 385     kvm => {
 386         optional => 1,
 387         type => 'boolean',
 388         description => "Enable/disable KVM hardware virtualization.",
 389         default => 1,
 390     },
 391     tdf => {
 392         optional => 1,
 393         type => 'boolean',
 394         description => "Enable/disable time drift fix.",
 395         default => 0,
 396     },
 397     localtime => {
 398         optional => 1,
 399         type => 'boolean',
 400         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 401     },
 402     freeze => {
 403         optional => 1,
 404         type => 'boolean',
 405         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 406     },
 407     vga => {
 408         optional => 1,
 409         type => 'string',
 410         description => "Select the VGA type.",
 411         verbose_description => "Select the VGA type. If you want to use high resolution" .
 412             " modes (>= 1280x1024x16) then you should use the options " .
 413             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 414             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 415             "display sever. For win* OS you can select how many independent " .
 416             "displays you want, Linux guests can add displays them self. " .
 417             "You can also run without any graphic card, using a serial device" .
 418             " as terminal.",
 419         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 420     },
 421     watchdog => {
 422         optional => 1,
 423         type => 'string', format => 'pve-qm-watchdog',
 424         description => "Create a virtual hardware watchdog device.",
 425         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 426             " (by a guest action), the watchdog must be periodically polled " .
 427             "by an agent inside the guest or else the watchdog will reset " .
 428             "the guest (or execute the respective action specified)",
 429     },
 430     startdate => {
 431         optional => 1,
 432         type => 'string',
 433         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 434         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 435         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 436         default => 'now',
 437     },
 438     startup =>  get_standard_option('pve-startup-order'),
 439     template => {
 440         optional => 1,
 441         type => 'boolean',
 442         description => "Enable/disable Template.",
 443         default => 0,
 444     },
 445     args => {
 446         optional => 1,
 447         type => 'string',
 448         description => "Arbitrary arguments passed to kvm.",
 449         verbose_description => <<EODESCR,
 450 Arbitrary arguments passed to kvm, for example:
 451
 452 args: -no-reboot -no-hpet
 453
 454 NOTE: this option is for experts only.
 455 EODESCR
 456     },
 457     tablet => {
 458         optional => 1,
 459         type => 'boolean',
 460         default => 1,
 461         description => "Enable/disable the USB tablet device.",
 462         verbose_description => "Enable/disable the USB tablet device. This device is " .
 463             "usually needed to allow absolute mouse positioning with VNC. " .
 464             "Else the mouse runs out of sync with normal VNC clients. " .
 465             "If you're running lots of console-only guests on one host, " .
 466             "you may consider disabling this to save some context switches. " .
 467             "This is turned off by default if you use spice (-vga=qxl).",
 468     },
 469     migrate_speed => {
 470         optional => 1,
 471         type => 'integer',
 472         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 473         minimum => 0,
 474         default => 0,
 475     },
 476     migrate_downtime => {
 477         optional => 1,
 478         type => 'number',
 479         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 480         minimum => 0,
 481         default => 0.1,
 482     },
 483     cdrom => {
 484         optional => 1,
 485         type => 'string', format => 'pve-qm-ide',
 486         typetext => '<volume>',
 487         description => "This is an alias for option -ide2",
 488     },
 489     cpu => {
 490         optional => 1,
 491         description => "Emulated CPU type.",
 492         type => 'string',
 493         format => $cpu_fmt,
 494     },
 495     parent => get_standard_option('pve-snapshot-name', {
 496         optional => 1,
 497         description => "Parent snapshot name. This is used internally, and should not be modified.",
 498     }),
 499     snaptime => {
 500         optional => 1,
 501         description => "Timestamp for snapshots.",
 502         type => 'integer',
 503         minimum => 0,
 504     },
 505     vmstate => {
 506         optional => 1,
 507         type => 'string', format => 'pve-volume-id',
 508         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 509     },
 510     vmstatestorage => get_standard_option('pve-storage-id', {
 511         description => "Default storage for VM state volumes/files.",
 512         optional => 1,
 513     }),
 514     machine => {
 515         description => "Specific the Qemu machine type.",
 516         type => 'string',
 517         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 518         maxLength => 40,
 519         optional => 1,
 520     },
 521     smbios1 => {
 522         description => "Specify SMBIOS type 1 fields.",
 523         type => 'string', format => 'pve-qm-smbios1',
 524         maxLength => 256,
 525         optional => 1,
 526     },
 527     protection => {
 528         optional => 1,
 529         type => 'boolean',
 530         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 531         default => 0,
 532     },
 533     bios => {
 534         optional => 1,
 535         type => 'string',
 536         enum => [ qw(seabios ovmf) ],
 537         description => "Select BIOS implementation.",
 538         default => 'seabios',
 539     },
 540 };
 541
 542 my $confdesc_cloudinit = {
 543     citype => {
 544         optional => 1,
 545         type => 'string',
 546         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 547         enum => ['configdrive2', 'nocloud'],
 548     },
 549     ciuser => {
 550         optional => 1,
 551         type => 'string',
 552         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 553     },
 554     cipassword => {
 555         optional => 1,
 556         type => 'string',
 557         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 558     },
 559     searchdomain => {
 560         optional => 1,
 561         type => 'string',
 562         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 563     },
 564     nameserver => {
 565         optional => 1,
 566         type => 'string', format => 'address-list',
 567         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 568     },
 569     sshkeys => {
 570         optional => 1,
 571         type => 'string',
 572         format => 'urlencoded',
 573         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 574     },
 575 };
 576
 577 # what about other qemu settings ?
 578 #cpu => 'string',
 579 #machine => 'string',
 580 #fda => 'file',
 581 #fdb => 'file',
 582 #mtdblock => 'file',
 583 #sd => 'file',
 584 #pflash => 'file',
 585 #snapshot => 'bool',
 586 #bootp => 'file',
 587 ##tftp => 'dir',
 588 ##smb => 'dir',
 589 #kernel => 'file',
 590 #append => 'string',
 591 #initrd => 'file',
 592 ##soundhw => 'string',
 593
 594 while (my ($k, $v) = each %$confdesc) {
 595     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 596 }
 597
 598 my $MAX_IDE_DISKS = 4;
 599 my $MAX_SCSI_DISKS = 14;
 600 my $MAX_VIRTIO_DISKS = 16;
 601 my $MAX_SATA_DISKS = 6;
 602 my $MAX_USB_DEVICES = 5;
 603 my $MAX_NETS = 32;
 604 my $MAX_UNUSED_DISKS = 8;
 605 my $MAX_HOSTPCI_DEVICES = 4;
 606 my $MAX_SERIAL_PORTS = 4;
 607 my $MAX_PARALLEL_PORTS = 3;
 608 my $MAX_NUMA = 8;
 609
 610 my $numa_fmt = {
 611     cpus => {
 612         type => "string",
 613         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 614         description => "CPUs accessing this NUMA node.",
 615         format_description => "id[-id];...",
 616     },
 617     memory => {
 618         type => "number",
 619         description => "Amount of memory this NUMA node provides.",
 620         optional => 1,
 621     },
 622     hostnodes => {
 623         type => "string",
 624         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 625         description => "Host NUMA nodes to use.",
 626         format_description => "id[-id];...",
 627         optional => 1,
 628     },
 629     policy => {
 630         type => 'string',
 631         enum => [qw(preferred bind interleave)],
 632         description => "NUMA allocation policy.",
 633         optional => 1,
 634     },
 635 };
 636 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 637 my $numadesc = {
 638     optional => 1,
 639     type => 'string', format => $numa_fmt,
 640     description => "NUMA topology.",
 641 };
 642 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 643
 644 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 645     $confdesc->{"numa$i"} = $numadesc;
 646 }
 647
 648 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 649                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 650                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 651 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 652
 653 my $net_fmt_bridge_descr = <<__EOD__;
 654 Bridge to attach the network device to. The Proxmox VE standard bridge
 655 is called 'vmbr0'.
 656
 657 If you do not specify a bridge, we create a kvm user (NATed) network
 658 device, which provides DHCP and DNS services. The following addresses
 659 are used:
 660
 661  10.0.2.2   Gateway
 662  10.0.2.3   DNS Server
 663  10.0.2.4   SMB Server
 664
 665 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 666 __EOD__
 667
 668 my $net_fmt = {
 669     macaddr => {
 670         type => 'string',
 671         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 672         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 673         format_description => "XX:XX:XX:XX:XX:XX",
 674         optional => 1,
 675     },
 676     model => {
 677         type => 'string',
 678         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 679         enum => $nic_model_list,
 680         default_key => 1,
 681     },
 682     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 683     bridge => {
 684         type => 'string',
 685         description => $net_fmt_bridge_descr,
 686         format_description => 'bridge',
 687         optional => 1,
 688     },
 689     queues => {
 690         type => 'integer',
 691         minimum => 0, maximum => 16,
 692         description => 'Number of packet queues to be used on the device.',
 693         optional => 1,
 694     },
 695     rate => {
 696         type => 'number',
 697         minimum => 0,
 698         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 699         optional => 1,
 700     },
 701     tag => {
 702         type => 'integer',
 703         minimum => 1, maximum => 4094,
 704         description => 'VLAN tag to apply to packets on this interface.',
 705         optional => 1,
 706     },
 707     trunks => {
 708         type => 'string',
 709         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 710         description => 'VLAN trunks to pass through this interface.',
 711         format_description => 'vlanid[;vlanid...]',
 712         optional => 1,
 713     },
 714     firewall => {
 715         type => 'boolean',
 716         description => 'Whether this interface should be protected by the firewall.',
 717         optional => 1,
 718     },
 719     link_down => {
 720         type => 'boolean',
 721         description => 'Whether this interface should be disconnected (like pulling the plug).',
 722         optional => 1,
 723     },
 724 };
 725
 726 my $netdesc = {
 727     optional => 1,
 728     type => 'string', format => $net_fmt,
 729     description => "Specify network devices.",
 730 };
 731
 732 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 733
 734 my $ipconfig_fmt = {
 735     ip => {
 736         type => 'string',
 737         format => 'pve-ipv4-config',
 738         format_description => 'IPv4Format/CIDR',
 739         description => 'IPv4 address in CIDR format.',
 740         optional => 1,
 741         default => 'dhcp',
 742     },
 743     gw => {
 744         type => 'string',
 745         format => 'ipv4',
 746         format_description => 'GatewayIPv4',
 747         description => 'Default gateway for IPv4 traffic.',
 748         optional => 1,
 749         requires => 'ip',
 750     },
 751     ip6 => {
 752         type => 'string',
 753         format => 'pve-ipv6-config',
 754         format_description => 'IPv6Format/CIDR',
 755         description => 'IPv6 address in CIDR format.',
 756         optional => 1,
 757         default => 'dhcp',
 758     },
 759     gw6 => {
 760         type => 'string',
 761         format => 'ipv6',
 762         format_description => 'GatewayIPv6',
 763         description => 'Default gateway for IPv6 traffic.',
 764         optional => 1,
 765         requires => 'ip6',
 766     },
 767 };
 768 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 769 my $ipconfigdesc = {
 770     optional => 1,
 771     type => 'string', format => 'pve-qm-ipconfig',
 772     description => <<'EODESCR',
 773 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 774
 775 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 776
 777 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 778 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 779
 780 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 781 EODESCR
 782 };
 783 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 784
 785 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 786     $confdesc->{"net$i"} = $netdesc;
 787     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 788 }
 789
 790 foreach my $key (keys %$confdesc_cloudinit) {
 791     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 792 }
 793
 794 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 795 sub verify_volume_id_or_qm_path {
 796     my ($volid, $noerr) = @_;
 797
 798     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 799         return $volid;
 800     }
 801
 802     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 803     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 804     if ($@) {
 805         return undef if $noerr;
 806         die $@;
 807     }
 808     return $volid;
 809 }
 810
 811 my $drivename_hash;
 812
 813 my %drivedesc_base = (
 814     volume => { alias => 'file' },
 815     file => {
 816         type => 'string',
 817         format => 'pve-volume-id-or-qm-path',
 818         default_key => 1,
 819         format_description => 'volume',
 820         description => "The drive's backing volume.",
 821     },
 822     media => {
 823         type => 'string',
 824         enum => [qw(cdrom disk)],
 825         description => "The drive's media type.",
 826         default => 'disk',
 827         optional => 1
 828     },
 829     cyls => {
 830         type => 'integer',
 831         description => "Force the drive's physical geometry to have a specific cylinder count.",
 832         optional => 1
 833     },
 834     heads => {
 835         type => 'integer',
 836         description => "Force the drive's physical geometry to have a specific head count.",
 837         optional => 1
 838     },
 839     secs => {
 840         type => 'integer',
 841         description => "Force the drive's physical geometry to have a specific sector count.",
 842         optional => 1
 843     },
 844     trans => {
 845         type => 'string',
 846         enum => [qw(none lba auto)],
 847         description => "Force disk geometry bios translation mode.",
 848         optional => 1,
 849     },
 850     snapshot => {
 851         type => 'boolean',
 852         description => "Controls qemu's snapshot mode feature."
 853             . " If activated, changes made to the disk are temporary and will"
 854             . " be discarded when the VM is shutdown.",
 855         optional => 1,
 856     },
 857     cache => {
 858         type => 'string',
 859         enum => [qw(none writethrough writeback unsafe directsync)],
 860         description => "The drive's cache mode",
 861         optional => 1,
 862     },
 863     format => get_standard_option('pve-qm-image-format'),
 864     size => {
 865         type => 'string',
 866         format => 'disk-size',
 867         format_description => 'DiskSize',
 868         description => "Disk size. This is purely informational and has no effect.",
 869         optional => 1,
 870     },
 871     backup => {
 872         type => 'boolean',
 873         description => "Whether the drive should be included when making backups.",
 874         optional => 1,
 875     },
 876     replicate => {
 877         type => 'boolean',
 878         description => 'Whether the drive should considered for replication jobs.',
 879         optional => 1,
 880         default => 1,
 881     },
 882     rerror => {
 883         type => 'string',
 884         enum => [qw(ignore report stop)],
 885         description => 'Read error action.',
 886         optional => 1,
 887     },
 888     werror => {
 889         type => 'string',
 890         enum => [qw(enospc ignore report stop)],
 891         description => 'Write error action.',
 892         optional => 1,
 893     },
 894     aio => {
 895         type => 'string',
 896         enum => [qw(native threads)],
 897         description => 'AIO type to use.',
 898         optional => 1,
 899     },
 900     discard => {
 901         type => 'string',
 902         enum => [qw(ignore on)],
 903         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 904         optional => 1,
 905     },
 906     detect_zeroes => {
 907         type => 'boolean',
 908         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 909         optional => 1,
 910     },
 911     serial => {
 912         type => 'string',
 913         format => 'urlencoded',
 914         format_description => 'serial',
 915         maxLength => 20*3, # *3 since it's %xx url enoded
 916         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 917         optional => 1,
 918     },
 919     shared => {
 920         type => 'boolean',
 921         description => 'Mark this locally-managed volume as available on all nodes',
 922         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 923         optional => 1,
 924         default => 0,
 925     }
 926 );
 927
 928 my %iothread_fmt = ( iothread => {
 929         type => 'boolean',
 930         description => "Whether to use iothreads for this drive",
 931         optional => 1,
 932 });
 933
 934 my %model_fmt = (
 935     model => {
 936         type => 'string',
 937         format => 'urlencoded',
 938         format_description => 'model',
 939         maxLength => 40*3, # *3 since it's %xx url enoded
 940         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 941         optional => 1,
 942     },
 943 );
 944
 945 my %queues_fmt = (
 946     queues => {
 947         type => 'integer',
 948         description => "Number of queues.",
 949         minimum => 2,
 950         optional => 1
 951     }
 952 );
 953
 954 my %scsiblock_fmt = (
 955     scsiblock => {
 956         type => 'boolean',
 957         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 958         optional => 1,
 959         default => 0,
 960     },
 961 );
 962
 963 my $add_throttle_desc = sub {
 964     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 965     my $d = {
 966         type => $type,
 967         format_description => $unit,
 968         description => "Maximum $what in $longunit.",
 969         optional => 1,
 970     };
 971     $d->{minimum} = $minimum if defined($minimum);
 972     $drivedesc_base{$key} = $d;
 973 };
 974 # throughput: (leaky bucket)
 975 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 976 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 977 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 978 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 979 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 980 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 981 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 982 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 983 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 984
 985 # pools: (pool of IO before throttling starts taking effect)
 986 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 987 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 988 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 989 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 990 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 991 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 992
 993 # burst lengths
 994 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 995 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 996 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 997 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 998 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 999 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1000
1001 # legacy support
1002 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1003 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1004 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1005 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1006
1007 my $ide_fmt = {
1008     %drivedesc_base,
1009     %model_fmt,
1010 };
1011 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1012
1013 my $idedesc = {
1014     optional => 1,
1015     type => 'string', format => $ide_fmt,
1016     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1017 };
1018 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1019
1020 my $scsi_fmt = {
1021     %drivedesc_base,
1022     %iothread_fmt,
1023     %queues_fmt,
1024     %scsiblock_fmt,
1025 };
1026 my $scsidesc = {
1027     optional => 1,
1028     type => 'string', format => $scsi_fmt,
1029     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1030 };
1031 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1032
1033 my $sata_fmt = {
1034     %drivedesc_base,
1035 };
1036 my $satadesc = {
1037     optional => 1,
1038     type => 'string', format => $sata_fmt,
1039     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1040 };
1041 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1042
1043 my $virtio_fmt = {
1044     %drivedesc_base,
1045     %iothread_fmt,
1046 };
1047 my $virtiodesc = {
1048     optional => 1,
1049     type => 'string', format => $virtio_fmt,
1050     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1051 };
1052 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1053
1054 my $alldrive_fmt = {
1055     %drivedesc_base,
1056     %iothread_fmt,
1057     %model_fmt,
1058     %queues_fmt,
1059     %scsiblock_fmt,
1060 };
1061
1062 my $efidisk_fmt = {
1063     volume => { alias => 'file' },
1064     file => {
1065         type => 'string',
1066         format => 'pve-volume-id-or-qm-path',
1067         default_key => 1,
1068         format_description => 'volume',
1069         description => "The drive's backing volume.",
1070     },
1071     format => get_standard_option('pve-qm-image-format'),
1072     size => {
1073         type => 'string',
1074         format => 'disk-size',
1075         format_description => 'DiskSize',
1076         description => "Disk size. This is purely informational and has no effect.",
1077         optional => 1,
1078     },
1079 };
1080
1081 my $efidisk_desc = {
1082     optional => 1,
1083     type => 'string', format => $efidisk_fmt,
1084     description => "Configure a Disk for storing EFI vars",
1085 };
1086
1087 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1088
1089 my $usb_fmt = {
1090     host => {
1091         default_key => 1,
1092         type => 'string', format => 'pve-qm-usb-device',
1093         format_description => 'HOSTUSBDEVICE|spice',
1094         description => <<EODESCR,
1095 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1096
1097  'bus-port(.port)*' (decimal numbers) or
1098  'vendor_id:product_id' (hexadeciaml numbers) or
1099  'spice'
1100
1101 You can use the 'lsusb -t' command to list existing usb devices.
1102
1103 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1104
1105 The value 'spice' can be used to add a usb redirection devices for spice.
1106 EODESCR
1107     },
1108     usb3 => {
1109         optional => 1,
1110         type => 'boolean',
1111         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1112         default => 0,
1113     },
1114 };
1115
1116 my $usbdesc = {
1117     optional => 1,
1118     type => 'string', format => $usb_fmt,
1119     description => "Configure an USB device (n is 0 to 4).",
1120 };
1121 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1122
1123 # NOTE: the match-groups of this regex are used in parse_hostpci
1124 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1125 my $hostpci_fmt = {
1126     host => {
1127         default_key => 1,
1128         type => 'string',
1129         pattern => qr/$PCIRE(;$PCIRE)*/,
1130         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1131         description => <<EODESCR,
1132 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1133 of PCI virtual functions of the host. HOSTPCIID syntax is:
1134
1135 'bus:dev.func' (hexadecimal numbers)
1136
1137 You can us the 'lspci' command to list existing PCI devices.
1138 EODESCR
1139     },
1140     rombar => {
1141         type => 'boolean',
1142         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1143         optional => 1,
1144         default => 1,
1145     },
1146     romfile => {
1147         type => 'string',
1148         pattern => '[^,;]+',
1149         format_description => 'string',
1150         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1151         optional => 1,
1152     },
1153     pcie => {
1154         type => 'boolean',
1155         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1156         optional => 1,
1157         default => 0,
1158     },
1159     'x-vga' => {
1160         type => 'boolean',
1161         description =>  "Enable vfio-vga device support.",
1162         optional => 1,
1163         default => 0,
1164     },
1165 };
1166 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1167
1168 my $hostpcidesc = {
1169         optional => 1,
1170         type => 'string', format => 'pve-qm-hostpci',
1171         description => "Map host PCI devices into guest.",
1172         verbose_description =>  <<EODESCR,
1173 Map host PCI devices into guest.
1174
1175 NOTE: This option allows direct access to host hardware. So it is no longer
1176 possible to migrate such machines - use with special care.
1177
1178 CAUTION: Experimental! User reported problems with this option.
1179 EODESCR
1180 };
1181 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1182
1183 my $serialdesc = {
1184         optional => 1,
1185         type => 'string',
1186         pattern => '(/dev/.+|socket)',
1187         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1188         verbose_description =>  <<EODESCR,
1189 Create a serial device inside the VM (n is 0 to 3), and pass through a
1190 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1191 host side (use 'qm terminal' to open a terminal connection).
1192
1193 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1194
1195 CAUTION: Experimental! User reported problems with this option.
1196 EODESCR
1197 };
1198
1199 my $paralleldesc= {
1200         optional => 1,
1201         type => 'string',
1202         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1203         description =>  "Map host parallel devices (n is 0 to 2).",
1204         verbose_description =>  <<EODESCR,
1205 Map host parallel devices (n is 0 to 2).
1206
1207 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1208
1209 CAUTION: Experimental! User reported problems with this option.
1210 EODESCR
1211 };
1212
1213 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1214     $confdesc->{"parallel$i"} = $paralleldesc;
1215 }
1216
1217 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1218     $confdesc->{"serial$i"} = $serialdesc;
1219 }
1220
1221 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1222     $confdesc->{"hostpci$i"} = $hostpcidesc;
1223 }
1224
1225 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1226     $drivename_hash->{"ide$i"} = 1;
1227     $confdesc->{"ide$i"} = $idedesc;
1228 }
1229
1230 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1231     $drivename_hash->{"sata$i"} = 1;
1232     $confdesc->{"sata$i"} = $satadesc;
1233 }
1234
1235 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1236     $drivename_hash->{"scsi$i"} = 1;
1237     $confdesc->{"scsi$i"} = $scsidesc ;
1238 }
1239
1240 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1241     $drivename_hash->{"virtio$i"} = 1;
1242     $confdesc->{"virtio$i"} = $virtiodesc;
1243 }
1244
1245 $drivename_hash->{efidisk0} = 1;
1246 $confdesc->{efidisk0} = $efidisk_desc;
1247
1248 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1249     $confdesc->{"usb$i"} = $usbdesc;
1250 }
1251
1252 my $unuseddesc = {
1253     optional => 1,
1254     type => 'string', format => 'pve-volume-id',
1255     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1256 };
1257
1258 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1259     $confdesc->{"unused$i"} = $unuseddesc;
1260 }
1261
1262 my $kvm_api_version = 0;
1263
1264 sub kvm_version {
1265
1266     return $kvm_api_version if $kvm_api_version;
1267
1268     my $fh = IO::File->new("</dev/kvm") ||
1269         return 0;
1270
1271     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1272         $kvm_api_version = $v;
1273     }
1274
1275     $fh->close();
1276
1277     return  $kvm_api_version;
1278 }
1279
1280 my $kvm_user_version;
1281
1282 sub kvm_user_version {
1283
1284     return $kvm_user_version if $kvm_user_version;
1285
1286     $kvm_user_version = 'unknown';
1287
1288     my $code = sub {
1289         my $line = shift;
1290         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1291             $kvm_user_version = $2;
1292         }
1293     };
1294
1295     eval { run_command("kvm -version", outfunc => $code); };
1296     warn $@ if $@;
1297
1298     return $kvm_user_version;
1299
1300 }
1301
1302 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1303
1304 sub valid_drive_names {
1305     # order is important - used to autoselect boot disk
1306     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1307             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1308             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1309             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1310             'efidisk0');
1311 }
1312
1313 sub is_valid_drivename {
1314     my $dev = shift;
1315
1316     return defined($drivename_hash->{$dev});
1317 }
1318
1319 sub option_exists {
1320     my $key = shift;
1321     return defined($confdesc->{$key});
1322 }
1323
1324 sub nic_models {
1325     return $nic_model_list;
1326 }
1327
1328 sub os_list_description {
1329
1330     return {
1331         other => 'Other',
1332         wxp => 'Windows XP',
1333         w2k => 'Windows 2000',
1334         w2k3 =>, 'Windows 2003',
1335         w2k8 => 'Windows 2008',
1336         wvista => 'Windows Vista',
1337         win7 => 'Windows 7',
1338         win8 => 'Windows 8/2012',
1339         win10 => 'Windows 10/2016',
1340         l24 => 'Linux 2.4',
1341         l26 => 'Linux 2.6',
1342     };
1343 }
1344
1345 my $cdrom_path;
1346
1347 sub get_cdrom_path {
1348
1349     return  $cdrom_path if $cdrom_path;
1350
1351     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1352     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1353     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1354 }
1355
1356 sub get_iso_path {
1357     my ($storecfg, $vmid, $cdrom) = @_;
1358
1359     if ($cdrom eq 'cdrom') {
1360         return get_cdrom_path();
1361     } elsif ($cdrom eq 'none') {
1362         return '';
1363     } elsif ($cdrom =~ m|^/|) {
1364         return $cdrom;
1365     } else {
1366         return PVE::Storage::path($storecfg, $cdrom);
1367     }
1368 }
1369
1370 # try to convert old style file names to volume IDs
1371 sub filename_to_volume_id {
1372     my ($vmid, $file, $media) = @_;
1373
1374      if (!($file eq 'none' || $file eq 'cdrom' ||
1375           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1376
1377         return undef if $file =~ m|/|;
1378
1379         if ($media && $media eq 'cdrom') {
1380             $file = "local:iso/$file";
1381         } else {
1382             $file = "local:$vmid/$file";
1383         }
1384     }
1385
1386     return $file;
1387 }
1388
1389 sub verify_media_type {
1390     my ($opt, $vtype, $media) = @_;
1391
1392     return if !$media;
1393
1394     my $etype;
1395     if ($media eq 'disk') {
1396         $etype = 'images';
1397     } elsif ($media eq 'cdrom') {
1398         $etype = 'iso';
1399     } else {
1400         die "internal error";
1401     }
1402
1403     return if ($vtype eq $etype);
1404
1405     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1406 }
1407
1408 sub cleanup_drive_path {
1409     my ($opt, $storecfg, $drive) = @_;
1410
1411     # try to convert filesystem paths to volume IDs
1412
1413     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1414         ($drive->{file} !~ m|^/dev/.+|) &&
1415         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1416         ($drive->{file} !~ m/^\d+$/)) {
1417         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1418         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1419         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1420         verify_media_type($opt, $vtype, $drive->{media});
1421         $drive->{file} = $volid;
1422     }
1423
1424     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1425 }
1426
1427 sub parse_hotplug_features {
1428     my ($data) = @_;
1429
1430     my $res = {};
1431
1432     return $res if $data eq '0';
1433
1434     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1435
1436     foreach my $feature (PVE::Tools::split_list($data)) {
1437         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1438             $res->{$1} = 1;
1439         } else {
1440             die "invalid hotplug feature '$feature'\n";
1441         }
1442     }
1443     return $res;
1444 }
1445
1446 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1447 sub pve_verify_hotplug_features {
1448     my ($value, $noerr) = @_;
1449
1450     return $value if parse_hotplug_features($value);
1451
1452     return undef if $noerr;
1453
1454     die "unable to parse hotplug option\n";
1455 }
1456
1457 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1458 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1459 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1460 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1461 #        [,iothread=on][,serial=serial][,model=model]
1462
1463 sub parse_drive {
1464     my ($key, $data) = @_;
1465
1466     my ($interface, $index);
1467
1468     if ($key =~ m/^([^\d]+)(\d+)$/) {
1469         $interface = $1;
1470         $index = $2;
1471     } else {
1472         return undef;
1473     }
1474
1475     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1476                                      : $confdesc->{$key}->{format};
1477     if (!$desc) {
1478         warn "invalid drive key: $key\n";
1479         return undef;
1480     }
1481     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1482     return undef if !$res;
1483     $res->{interface} = $interface;
1484     $res->{index} = $index;
1485
1486     my $error = 0;
1487     foreach my $opt (qw(bps bps_rd bps_wr)) {
1488         if (my $bps = defined(delete $res->{$opt})) {
1489             if (defined($res->{"m$opt"})) {
1490                 warn "both $opt and m$opt specified\n";
1491                 ++$error;
1492                 next;
1493             }
1494             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1495         }
1496     }
1497
1498     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1499     for my $requirement (
1500         [mbps_max => 'mbps'],
1501         [mbps_rd_max => 'mbps_rd'],
1502         [mbps_wr_max => 'mbps_wr'],
1503         [miops_max => 'miops'],
1504         [miops_rd_max => 'miops_rd'],
1505         [miops_wr_max => 'miops_wr'],
1506         [bps_max_length => 'mbps_max'],
1507         [bps_rd_max_length => 'mbps_rd_max'],
1508         [bps_wr_max_length => 'mbps_wr_max'],
1509         [iops_max_length => 'iops_max'],
1510         [iops_rd_max_length => 'iops_rd_max'],
1511         [iops_wr_max_length => 'iops_wr_max']) {
1512         my ($option, $requires) = @$requirement;
1513         if ($res->{$option} && !$res->{$requires}) {
1514             warn "$option requires $requires\n";
1515             ++$error;
1516         }
1517     }
1518
1519     return undef if $error;
1520
1521     return undef if $res->{mbps_rd} && $res->{mbps};
1522     return undef if $res->{mbps_wr} && $res->{mbps};
1523     return undef if $res->{iops_rd} && $res->{iops};
1524     return undef if $res->{iops_wr} && $res->{iops};
1525
1526     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1527         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1528         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1529         return undef if $res->{interface} eq 'virtio';
1530     }
1531
1532     if (my $size = $res->{size}) {
1533         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1534     }
1535
1536     return $res;
1537 }
1538
1539 sub print_drive {
1540     my ($vmid, $drive) = @_;
1541     my $data = { %$drive };
1542     delete $data->{$_} for qw(index interface);
1543     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1544 }
1545
1546 sub scsi_inquiry {
1547     my($fh, $noerr) = @_;
1548
1549     my $SG_IO = 0x2285;
1550     my $SG_GET_VERSION_NUM = 0x2282;
1551
1552     my $versionbuf = "\x00" x 8;
1553     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1554     if (!$ret) {
1555         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1556         return undef;
1557     }
1558     my $version = unpack("I", $versionbuf);
1559     if ($version < 30000) {
1560         die "scsi generic interface too old\n"  if !$noerr;
1561         return undef;
1562     }
1563
1564     my $buf = "\x00" x 36;
1565     my $sensebuf = "\x00" x 8;
1566     my $cmd = pack("C x3 C x1", 0x12, 36);
1567
1568     # see /usr/include/scsi/sg.h
1569     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1570
1571     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1572                       length($sensebuf), 0, length($buf), $buf,
1573                       $cmd, $sensebuf, 6000);
1574
1575     $ret = ioctl($fh, $SG_IO, $packet);
1576     if (!$ret) {
1577         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1578         return undef;
1579     }
1580
1581     my @res = unpack($sg_io_hdr_t, $packet);
1582     if ($res[17] || $res[18]) {
1583         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1584         return undef;
1585     }
1586
1587     my $res = {};
1588     (my $byte0, my $byte1, $res->{vendor},
1589      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1590
1591     $res->{removable} = $byte1 & 128 ? 1 : 0;
1592     $res->{type} = $byte0 & 31;
1593
1594     return $res;
1595 }
1596
1597 sub path_is_scsi {
1598     my ($path) = @_;
1599
1600     my $fh = IO::File->new("+<$path") || return undef;
1601     my $res = scsi_inquiry($fh, 1);
1602     close($fh);
1603
1604     return $res;
1605 }
1606
1607 sub machine_type_is_q35 {
1608     my ($conf) = @_;
1609
1610     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1611 }
1612
1613 sub print_tabletdevice_full {
1614     my ($conf) = @_;
1615
1616     my $q35 = machine_type_is_q35($conf);
1617
1618     # we use uhci for old VMs because tablet driver was buggy in older qemu
1619     my $usbbus = $q35 ? "ehci" : "uhci";
1620
1621     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1622 }
1623
1624 sub print_drivedevice_full {
1625     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1626
1627     my $device = '';
1628     my $maxdev = 0;
1629
1630     if ($drive->{interface} eq 'virtio') {
1631         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1632         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1633         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1634     } elsif ($drive->{interface} eq 'scsi') {
1635
1636         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1637         my $unit = $drive->{index} % $maxdev;
1638         my $devicetype = 'hd';
1639         my $path = '';
1640         if (drive_is_cdrom($drive)) {
1641             $devicetype = 'cd';
1642         } else {
1643             if ($drive->{file} =~ m|^/|) {
1644                 $path = $drive->{file};
1645                 if (my $info = path_is_scsi($path)) {
1646                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1647                         $devicetype = 'block';
1648                     } elsif ($info->{type} == 1) { # tape
1649                         $devicetype = 'generic';
1650                     }
1651                 }
1652             } else {
1653                  $path = PVE::Storage::path($storecfg, $drive->{file});
1654             }
1655
1656             if($path =~ m/^iscsi\:\/\//){
1657                 $devicetype = 'generic';
1658             }
1659         }
1660
1661         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1662             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1663         } else {
1664             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1665         }
1666
1667     } elsif ($drive->{interface} eq 'ide'){
1668         $maxdev = 2;
1669         my $controller = int($drive->{index} / $maxdev);
1670         my $unit = $drive->{index} % $maxdev;
1671         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1672
1673         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1674         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1675             $model = URI::Escape::uri_unescape($model);
1676             $device .= ",model=$model";
1677         }
1678     } elsif ($drive->{interface} eq 'sata'){
1679         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1680         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1681         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1682     } elsif ($drive->{interface} eq 'usb') {
1683         die "implement me";
1684         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1685     } else {
1686         die "unsupported interface type";
1687     }
1688
1689     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1690
1691     return $device;
1692 }
1693
1694 sub get_initiator_name {
1695     my $initiator;
1696
1697     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1698     while (defined(my $line = <$fh>)) {
1699         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1700         $initiator = $1;
1701         last;
1702     }
1703     $fh->close();
1704
1705     return $initiator;
1706 }
1707
1708 sub print_drive_full {
1709     my ($storecfg, $vmid, $drive) = @_;
1710
1711     my $path;
1712     my $volid = $drive->{file};
1713     my $format;
1714
1715     if (drive_is_cdrom($drive)) {
1716         $path = get_iso_path($storecfg, $vmid, $volid);
1717     } else {
1718         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1719         if ($storeid) {
1720             $path = PVE::Storage::path($storecfg, $volid);
1721             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1722             $format = qemu_img_format($scfg, $volname);
1723         } else {
1724             $path = $volid;
1725             $format = "raw";
1726         }
1727    }
1728
1729     my $opts = '';
1730     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1731     foreach my $o (@qemu_drive_options) {
1732         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1733     }
1734
1735     # snapshot only accepts on|off
1736     if (defined($drive->{snapshot})) {
1737         my $v = $drive->{snapshot} ? 'on' : 'off';
1738         $opts .= ",snapshot=$v";
1739     }
1740
1741     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1742         my ($dir, $qmpname) = @$type;
1743         if (my $v = $drive->{"mbps$dir"}) {
1744             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1745         }
1746         if (my $v = $drive->{"mbps${dir}_max"}) {
1747             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1748         }
1749         if (my $v = $drive->{"bps${dir}_max_length"}) {
1750             $opts .= ",throttling.bps$qmpname-max-length=$v";
1751         }
1752         if (my $v = $drive->{"iops${dir}"}) {
1753             $opts .= ",throttling.iops$qmpname=$v";
1754         }
1755         if (my $v = $drive->{"iops${dir}_max"}) {
1756             $opts .= ",throttling.iops$qmpname-max=$v";
1757         }
1758         if (my $v = $drive->{"iops${dir}_max_length"}) {
1759             $opts .= ",throttling.iops$qmpname-max-length=$v";
1760         }
1761     }
1762
1763     if (my $serial = $drive->{serial}) {
1764         $serial = URI::Escape::uri_unescape($serial);
1765         $opts .= ",serial=$serial";
1766     }
1767
1768     $opts .= ",format=$format" if $format && !$drive->{format};
1769
1770     my $cache_direct = 0;
1771
1772     if (my $cache = $drive->{cache}) {
1773         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1774     } elsif (!drive_is_cdrom($drive)) {
1775         $opts .= ",cache=none";
1776         $cache_direct = 1;
1777     }
1778
1779     # aio native works only with O_DIRECT
1780     if (!$drive->{aio}) {
1781         if($cache_direct) {
1782             $opts .= ",aio=native";
1783         } else {
1784             $opts .= ",aio=threads";
1785         }
1786     }
1787
1788     if (!drive_is_cdrom($drive)) {
1789         my $detectzeroes;
1790         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1791             $detectzeroes = 'off';
1792         } elsif ($drive->{discard}) {
1793             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1794         } else {
1795             # This used to be our default with discard not being specified:
1796             $detectzeroes = 'on';
1797         }
1798         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1799     }
1800
1801     my $pathinfo = $path ? "file=$path," : '';
1802
1803     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1804 }
1805
1806 sub print_netdevice_full {
1807     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1808
1809     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1810
1811     my $device = $net->{model};
1812     if ($net->{model} eq 'virtio') {
1813          $device = 'virtio-net-pci';
1814      };
1815
1816     my $pciaddr = print_pci_addr("$netid", $bridges);
1817     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1818     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1819         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1820         my $vectors = $net->{queues} * 2 + 2;
1821         $tmpstr .= ",vectors=$vectors,mq=on";
1822     }
1823     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1824
1825     if ($use_old_bios_files) {
1826         my $romfile;
1827         if ($device eq 'virtio-net-pci') {
1828             $romfile = 'pxe-virtio.rom';
1829         } elsif ($device eq 'e1000') {
1830             $romfile = 'pxe-e1000.rom';
1831         } elsif ($device eq 'ne2k') {
1832             $romfile = 'pxe-ne2k_pci.rom';
1833         } elsif ($device eq 'pcnet') {
1834             $romfile = 'pxe-pcnet.rom';
1835         } elsif ($device eq 'rtl8139') {
1836             $romfile = 'pxe-rtl8139.rom';
1837         }
1838         $tmpstr .= ",romfile=$romfile" if $romfile;
1839     }
1840
1841     return $tmpstr;
1842 }
1843
1844 sub print_netdev_full {
1845     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1846
1847     my $i = '';
1848     if ($netid =~ m/^net(\d+)$/) {
1849         $i = int($1);
1850     }
1851
1852     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1853
1854     my $ifname = "tap${vmid}i$i";
1855
1856     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1857     die "interface name '$ifname' is too long (max 15 character)\n"
1858         if length($ifname) >= 16;
1859
1860     my $vhostparam = '';
1861     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1862
1863     my $vmname = $conf->{name} || "vm$vmid";
1864
1865     my $netdev = "";
1866     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1867
1868     if ($net->{bridge}) {
1869         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1870     } else {
1871         $netdev = "type=user,id=$netid,hostname=$vmname";
1872     }
1873
1874     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1875
1876     return $netdev;
1877 }
1878
1879
1880 sub print_cpu_device {
1881     my ($conf, $id) = @_;
1882
1883     my $kvm = $conf->{kvm} // 1;
1884     my $cpu = $kvm ? "kvm64" : "qemu64";
1885     if (my $cputype = $conf->{cpu}) {
1886         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1887             or die "Cannot parse cpu description: $cputype\n";
1888         $cpu = $cpuconf->{cputype};
1889     }
1890
1891     my $cores = $conf->{cores} || 1;
1892
1893     my $current_core = ($id - 1) % $cores;
1894     my $current_socket = int(($id - 1 - $current_core)/$cores);
1895
1896     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1897 }
1898
1899 sub drive_is_cloudinit {
1900     my ($drive) = @_;
1901     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
1902 }
1903
1904 sub drive_is_cdrom {
1905     my ($drive, $exclude_cloudinit) = @_;
1906
1907     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
1908
1909     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1910
1911 }
1912
1913 sub parse_number_sets {
1914     my ($set) = @_;
1915     my $res = [];
1916     foreach my $part (split(/;/, $set)) {
1917         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1918             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1919             push @$res, [ $1, $2 ];
1920         } else {
1921             die "invalid range: $part\n";
1922         }
1923     }
1924     return $res;
1925 }
1926
1927 sub parse_numa {
1928     my ($data) = @_;
1929
1930     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1931     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1932     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1933     return $res;
1934 }
1935
1936 sub parse_hostpci {
1937     my ($value) = @_;
1938
1939     return undef if !$value;
1940
1941     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1942
1943     my @idlist = split(/;/, $res->{host});
1944     delete $res->{host};
1945     foreach my $id (@idlist) {
1946         if ($id =~ /^$PCIRE$/) {
1947             if (defined($2)) {
1948                 push @{$res->{pciid}}, { id => $1, function => $2 };
1949             } else {
1950                 my $pcidevices = lspci($1);
1951                 $res->{pciid} = $pcidevices->{$1};
1952             }
1953         } else {
1954             # should have been caught by parse_property_string already
1955             die "failed to parse PCI id: $id\n";
1956         }
1957     }
1958     return $res;
1959 }
1960
1961 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1962 sub parse_net {
1963     my ($data) = @_;
1964
1965     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1966     if ($@) {
1967         warn $@;
1968         return undef;
1969     }
1970     if (!defined($res->{macaddr})) {
1971         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1972         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1973     }
1974     return $res;
1975 }
1976
1977 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1978 sub parse_ipconfig {
1979     my ($data) = @_;
1980
1981     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1982     if ($@) {
1983         warn $@;
1984         return undef;
1985     }
1986
1987     if ($res->{gw} && !$res->{ip}) {
1988         warn 'gateway specified without specifying an IP address';
1989         return undef;
1990     }
1991     if ($res->{gw6} && !$res->{ip6}) {
1992         warn 'IPv6 gateway specified without specifying an IPv6 address';
1993         return undef;
1994     }
1995     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1996         warn 'gateway specified together with DHCP';
1997         return undef;
1998     }
1999     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2000         # gw6 + auto/dhcp
2001         warn "IPv6 gateway specified together with $res->{ip6} address";
2002         return undef;
2003     }
2004
2005     if (!$res->{ip} && !$res->{ip6}) {
2006         return { ip => 'dhcp', ip6 => 'dhcp' };
2007     }
2008
2009     return $res;
2010 }
2011
2012 sub print_net {
2013     my $net = shift;
2014
2015     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2016 }
2017
2018 sub add_random_macs {
2019     my ($settings) = @_;
2020
2021     foreach my $opt (keys %$settings) {
2022         next if $opt !~ m/^net(\d+)$/;
2023         my $net = parse_net($settings->{$opt});
2024         next if !$net;
2025         $settings->{$opt} = print_net($net);
2026     }
2027 }
2028
2029 sub vm_is_volid_owner {
2030     my ($storecfg, $vmid, $volid) = @_;
2031
2032     if ($volid !~  m|^/|) {
2033         my ($path, $owner);
2034         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2035         if ($owner && ($owner == $vmid)) {
2036             return 1;
2037         }
2038     }
2039
2040     return undef;
2041 }
2042
2043 sub split_flagged_list {
2044     my $text = shift || '';
2045     $text =~ s/[,;]/ /g;
2046     $text =~ s/^\s+//;
2047     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2048 }
2049
2050 sub join_flagged_list {
2051     my ($how, $lst) = @_;
2052     join $how, map { $lst->{$_} . $_ } keys %$lst;
2053 }
2054
2055 sub vmconfig_delete_pending_option {
2056     my ($conf, $key, $force) = @_;
2057
2058     delete $conf->{pending}->{$key};
2059     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2060     $pending_delete_hash->{$key} = $force ? '!' : '';
2061     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2062 }
2063
2064 sub vmconfig_undelete_pending_option {
2065     my ($conf, $key) = @_;
2066
2067     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2068     delete $pending_delete_hash->{$key};
2069
2070     if (%$pending_delete_hash) {
2071         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2072     } else {
2073         delete $conf->{pending}->{delete};
2074     }
2075 }
2076
2077 sub vmconfig_register_unused_drive {
2078     my ($storecfg, $vmid, $conf, $drive) = @_;
2079
2080     if (drive_is_cloudinit($drive)) {
2081         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2082         warn $@ if $@;
2083     } elsif (!drive_is_cdrom($drive)) {
2084         my $volid = $drive->{file};
2085         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2086             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2087         }
2088     }
2089 }
2090
2091 sub vmconfig_cleanup_pending {
2092     my ($conf) = @_;
2093
2094     # remove pending changes when nothing changed
2095     my $changes;
2096     foreach my $opt (keys %{$conf->{pending}}) {
2097         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2098             $changes = 1;
2099             delete $conf->{pending}->{$opt};
2100         }
2101     }
2102
2103     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2104     my $pending_delete_hash = {};
2105     while (my ($opt, $force) = each %$current_delete_hash) {
2106         if (defined($conf->{$opt})) {
2107             $pending_delete_hash->{$opt} = $force;
2108         } else {
2109             $changes = 1;
2110         }
2111     }
2112
2113     if (%$pending_delete_hash) {
2114         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2115     } else {
2116         delete $conf->{pending}->{delete};
2117     }
2118
2119     return $changes;
2120 }
2121
2122 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2123 my $smbios1_fmt = {
2124     uuid => {
2125         type => 'string',
2126         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2127         format_description => 'UUID',
2128         description => "Set SMBIOS1 UUID.",
2129         optional => 1,
2130     },
2131     version => {
2132         type => 'string',
2133         pattern => '\S+',
2134         format_description => 'string',
2135         description => "Set SMBIOS1 version.",
2136         optional => 1,
2137     },
2138     serial => {
2139         type => 'string',
2140         pattern => '\S+',
2141         format_description => 'string',
2142         description => "Set SMBIOS1 serial number.",
2143         optional => 1,
2144     },
2145     manufacturer => {
2146         type => 'string',
2147         pattern => '\S+',
2148         format_description => 'string',
2149         description => "Set SMBIOS1 manufacturer.",
2150         optional => 1,
2151     },
2152     product => {
2153         type => 'string',
2154         pattern => '\S+',
2155         format_description => 'string',
2156         description => "Set SMBIOS1 product ID.",
2157         optional => 1,
2158     },
2159     sku => {
2160         type => 'string',
2161         pattern => '\S+',
2162         format_description => 'string',
2163         description => "Set SMBIOS1 SKU string.",
2164         optional => 1,
2165     },
2166     family => {
2167         type => 'string',
2168         pattern => '\S+',
2169         format_description => 'string',
2170         description => "Set SMBIOS1 family string.",
2171         optional => 1,
2172     },
2173 };
2174
2175 sub parse_smbios1 {
2176     my ($data) = @_;
2177
2178     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2179     warn $@ if $@;
2180     return $res;
2181 }
2182
2183 sub print_smbios1 {
2184     my ($smbios1) = @_;
2185     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2186 }
2187
2188 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2189
2190 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2191 sub verify_bootdisk {
2192     my ($value, $noerr) = @_;
2193
2194     return $value if is_valid_drivename($value);
2195
2196     return undef if $noerr;
2197
2198     die "invalid boot disk '$value'\n";
2199 }
2200
2201 sub parse_watchdog {
2202     my ($value) = @_;
2203
2204     return undef if !$value;
2205
2206     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2207     warn $@ if $@;
2208     return $res;
2209 }
2210
2211 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2212 sub verify_usb_device {
2213     my ($value, $noerr) = @_;
2214
2215     return $value if parse_usb_device($value);
2216
2217     return undef if $noerr;
2218
2219     die "unable to parse usb device\n";
2220 }
2221
2222 # add JSON properties for create and set function
2223 sub json_config_properties {
2224     my $prop = shift;
2225
2226     foreach my $opt (keys %$confdesc) {
2227         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2228         $prop->{$opt} = $confdesc->{$opt};
2229     }
2230
2231     return $prop;
2232 }
2233
2234 # return copy of $confdesc_cloudinit to generate documentation
2235 sub cloudinit_config_properties {
2236
2237     return dclone($confdesc_cloudinit);
2238 }
2239
2240 sub check_type {
2241     my ($key, $value) = @_;
2242
2243     die "unknown setting '$key'\n" if !$confdesc->{$key};
2244
2245     my $type = $confdesc->{$key}->{type};
2246
2247     if (!defined($value)) {
2248         die "got undefined value\n";
2249     }
2250
2251     if ($value =~ m/[\n\r]/) {
2252         die "property contains a line feed\n";
2253     }
2254
2255     if ($type eq 'boolean') {
2256         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2257         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2258         die "type check ('boolean') failed - got '$value'\n";
2259     } elsif ($type eq 'integer') {
2260         return int($1) if $value =~ m/^(\d+)$/;
2261         die "type check ('integer') failed - got '$value'\n";
2262     } elsif ($type eq 'number') {
2263         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2264         die "type check ('number') failed - got '$value'\n";
2265     } elsif ($type eq 'string') {
2266         if (my $fmt = $confdesc->{$key}->{format}) {
2267             PVE::JSONSchema::check_format($fmt, $value);
2268             return $value;
2269         }
2270         $value =~ s/^\"(.*)\"$/$1/;
2271         return $value;
2272     } else {
2273         die "internal error"
2274     }
2275 }
2276
2277 sub check_iommu_support{
2278     #fixme : need to check IOMMU support
2279     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2280
2281     my $iommu=1;
2282     return $iommu;
2283
2284 }
2285
2286 sub touch_config {
2287     my ($vmid) = @_;
2288
2289     my $conf = PVE::QemuConfig->config_file($vmid);
2290     utime undef, undef, $conf;
2291 }
2292
2293 sub destroy_vm {
2294     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2295
2296     my $conffile = PVE::QemuConfig->config_file($vmid);
2297
2298     my $conf = PVE::QemuConfig->load_config($vmid);
2299
2300     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2301
2302     if ($conf->{template}) {
2303         # check if any base image is still used by a linked clone
2304         foreach_drive($conf, sub {
2305                 my ($ds, $drive) = @_;
2306
2307                 return if drive_is_cdrom($drive);
2308
2309                 my $volid = $drive->{file};
2310
2311                 return if !$volid || $volid =~ m|^/|;
2312
2313                 die "base volume '$volid' is still in use by linked cloned\n"
2314                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2315
2316         });
2317     }
2318
2319     # only remove disks owned by this VM
2320     foreach_drive($conf, sub {
2321         my ($ds, $drive) = @_;
2322
2323         return if drive_is_cdrom($drive, 1);
2324
2325         my $volid = $drive->{file};
2326
2327         return if !$volid || $volid =~ m|^/|;
2328
2329         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2330         return if !$path || !$owner || ($owner != $vmid);
2331
2332         eval {
2333             PVE::Storage::vdisk_free($storecfg, $volid);
2334         };
2335         warn "Could not remove disk '$volid', check manually: $@" if $@;
2336
2337     });
2338
2339     if ($keep_empty_config) {
2340         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2341     } else {
2342         unlink $conffile;
2343     }
2344
2345     # also remove unused disk
2346     eval {
2347         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2348
2349         eval {
2350             PVE::Storage::foreach_volid($dl, sub {
2351                 my ($volid, $sid, $volname, $d) = @_;
2352                 PVE::Storage::vdisk_free($storecfg, $volid);
2353             });
2354         };
2355         warn $@ if $@;
2356
2357     };
2358     warn $@ if $@;
2359 }
2360
2361 sub parse_vm_config {
2362     my ($filename, $raw) = @_;
2363
2364     return undef if !defined($raw);
2365
2366     my $res = {
2367         digest => Digest::SHA::sha1_hex($raw),
2368         snapshots => {},
2369         pending => {},
2370     };
2371
2372     $filename =~ m|/qemu-server/(\d+)\.conf$|
2373         || die "got strange filename '$filename'";
2374
2375     my $vmid = $1;
2376
2377     my $conf = $res;
2378     my $descr;
2379     my $section = '';
2380
2381     my @lines = split(/\n/, $raw);
2382     foreach my $line (@lines) {
2383         next if $line =~ m/^\s*$/;
2384
2385         if ($line =~ m/^\[PENDING\]\s*$/i) {
2386             $section = 'pending';
2387             if (defined($descr)) {
2388                 $descr =~ s/\s+$//;
2389                 $conf->{description} = $descr;
2390             }
2391             $descr = undef;
2392             $conf = $res->{$section} = {};
2393             next;
2394
2395         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2396             $section = $1;
2397             if (defined($descr)) {
2398                 $descr =~ s/\s+$//;
2399                 $conf->{description} = $descr;
2400             }
2401             $descr = undef;
2402             $conf = $res->{snapshots}->{$section} = {};
2403             next;
2404         }
2405
2406         if ($line =~ m/^\#(.*)\s*$/) {
2407             $descr = '' if !defined($descr);
2408             $descr .= PVE::Tools::decode_text($1) . "\n";
2409             next;
2410         }
2411
2412         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2413             $descr = '' if !defined($descr);
2414             $descr .= PVE::Tools::decode_text($2);
2415         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2416             $conf->{snapstate} = $1;
2417         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2418             my $key = $1;
2419             my $value = $2;
2420             $conf->{$key} = $value;
2421         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2422             my $value = $1;
2423             if ($section eq 'pending') {
2424                 $conf->{delete} = $value; # we parse this later
2425             } else {
2426                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2427             }
2428         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2429             my $key = $1;
2430             my $value = $2;
2431             eval { $value = check_type($key, $value); };
2432             if ($@) {
2433                 warn "vm $vmid - unable to parse value of '$key' - $@";
2434             } else {
2435                 $key = 'ide2' if $key eq 'cdrom';
2436                 my $fmt = $confdesc->{$key}->{format};
2437                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2438                     my $v = parse_drive($key, $value);
2439                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2440                         $v->{file} = $volid;
2441                         $value = print_drive($vmid, $v);
2442                     } else {
2443                         warn "vm $vmid - unable to parse value of '$key'\n";
2444                         next;
2445                     }
2446                 }
2447
2448                 $conf->{$key} = $value;
2449             }
2450         }
2451     }
2452
2453     if (defined($descr)) {
2454         $descr =~ s/\s+$//;
2455         $conf->{description} = $descr;
2456     }
2457     delete $res->{snapstate}; # just to be sure
2458
2459     return $res;
2460 }
2461
2462 sub write_vm_config {
2463     my ($filename, $conf) = @_;
2464
2465     delete $conf->{snapstate}; # just to be sure
2466
2467     if ($conf->{cdrom}) {
2468         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2469         $conf->{ide2} = $conf->{cdrom};
2470         delete $conf->{cdrom};
2471     }
2472
2473     # we do not use 'smp' any longer
2474     if ($conf->{sockets}) {
2475         delete $conf->{smp};
2476     } elsif ($conf->{smp}) {
2477         $conf->{sockets} = $conf->{smp};
2478         delete $conf->{cores};
2479         delete $conf->{smp};
2480     }
2481
2482     my $used_volids = {};
2483
2484     my $cleanup_config = sub {
2485         my ($cref, $pending, $snapname) = @_;
2486
2487         foreach my $key (keys %$cref) {
2488             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2489                 $key eq 'snapstate' || $key eq 'pending';
2490             my $value = $cref->{$key};
2491             if ($key eq 'delete') {
2492                 die "propertry 'delete' is only allowed in [PENDING]\n"
2493                     if !$pending;
2494                 # fixme: check syntax?
2495                 next;
2496             }
2497             eval { $value = check_type($key, $value); };
2498             die "unable to parse value of '$key' - $@" if $@;
2499
2500             $cref->{$key} = $value;
2501
2502             if (!$snapname && is_valid_drivename($key)) {
2503                 my $drive = parse_drive($key, $value);
2504                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2505             }
2506         }
2507     };
2508
2509     &$cleanup_config($conf);
2510
2511     &$cleanup_config($conf->{pending}, 1);
2512
2513     foreach my $snapname (keys %{$conf->{snapshots}}) {
2514         die "internal error" if $snapname eq 'pending';
2515         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2516     }
2517
2518     # remove 'unusedX' settings if we re-add a volume
2519     foreach my $key (keys %$conf) {
2520         my $value = $conf->{$key};
2521         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2522             delete $conf->{$key};
2523         }
2524     }
2525
2526     my $generate_raw_config = sub {
2527         my ($conf, $pending) = @_;
2528
2529         my $raw = '';
2530
2531         # add description as comment to top of file
2532         if (defined(my $descr = $conf->{description})) {
2533             if ($descr) {
2534                 foreach my $cl (split(/\n/, $descr)) {
2535                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2536                 }
2537             } else {
2538                 $raw .= "#\n" if $pending;
2539             }
2540         }
2541
2542         foreach my $key (sort keys %$conf) {
2543             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2544             $raw .= "$key: $conf->{$key}\n";
2545         }
2546         return $raw;
2547     };
2548
2549     my $raw = &$generate_raw_config($conf);
2550
2551     if (scalar(keys %{$conf->{pending}})){
2552         $raw .= "\n[PENDING]\n";
2553         $raw .= &$generate_raw_config($conf->{pending}, 1);
2554     }
2555
2556     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2557         $raw .= "\n[$snapname]\n";
2558         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2559     }
2560
2561     return $raw;
2562 }
2563
2564 sub load_defaults {
2565
2566     my $res = {};
2567
2568     # we use static defaults from our JSON schema configuration
2569     foreach my $key (keys %$confdesc) {
2570         if (defined(my $default = $confdesc->{$key}->{default})) {
2571             $res->{$key} = $default;
2572         }
2573     }
2574
2575     return $res;
2576 }
2577
2578 sub config_list {
2579     my $vmlist = PVE::Cluster::get_vmlist();
2580     my $res = {};
2581     return $res if !$vmlist || !$vmlist->{ids};
2582     my $ids = $vmlist->{ids};
2583
2584     foreach my $vmid (keys %$ids) {
2585         my $d = $ids->{$vmid};
2586         next if !$d->{node} || $d->{node} ne $nodename;
2587         next if !$d->{type} || $d->{type} ne 'qemu';
2588         $res->{$vmid}->{exists} = 1;
2589     }
2590     return $res;
2591 }
2592
2593 # test if VM uses local resources (to prevent migration)
2594 sub check_local_resources {
2595     my ($conf, $noerr) = @_;
2596
2597     my $loc_res = 0;
2598
2599     $loc_res = 1 if $conf->{hostusb}; # old syntax
2600     $loc_res = 1 if $conf->{hostpci}; # old syntax
2601
2602     foreach my $k (keys %$conf) {
2603         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2604         # sockets are safe: they will recreated be on the target side post-migrate
2605         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2606         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2607     }
2608
2609     die "VM uses local resources\n" if $loc_res && !$noerr;
2610
2611     return $loc_res;
2612 }
2613
2614 # check if used storages are available on all nodes (use by migrate)
2615 sub check_storage_availability {
2616     my ($storecfg, $conf, $node) = @_;
2617
2618     foreach_drive($conf, sub {
2619         my ($ds, $drive) = @_;
2620
2621         my $volid = $drive->{file};
2622         return if !$volid;
2623
2624         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2625         return if !$sid;
2626
2627         # check if storage is available on both nodes
2628         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2629         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2630    });
2631 }
2632
2633 # list nodes where all VM images are available (used by has_feature API)
2634 sub shared_nodes {
2635     my ($conf, $storecfg) = @_;
2636
2637     my $nodelist = PVE::Cluster::get_nodelist();
2638     my $nodehash = { map { $_ => 1 } @$nodelist };
2639     my $nodename = PVE::INotify::nodename();
2640
2641     foreach_drive($conf, sub {
2642         my ($ds, $drive) = @_;
2643
2644         my $volid = $drive->{file};
2645         return if !$volid;
2646
2647         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2648         if ($storeid) {
2649             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2650             if ($scfg->{disable}) {
2651                 $nodehash = {};
2652             } elsif (my $avail = $scfg->{nodes}) {
2653                 foreach my $node (keys %$nodehash) {
2654                     delete $nodehash->{$node} if !$avail->{$node};
2655                 }
2656             } elsif (!$scfg->{shared}) {
2657                 foreach my $node (keys %$nodehash) {
2658                     delete $nodehash->{$node} if $node ne $nodename
2659                 }
2660             }
2661         }
2662     });
2663
2664     return $nodehash
2665 }
2666
2667 sub check_cmdline {
2668     my ($pidfile, $pid) = @_;
2669
2670     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2671     if (defined($fh)) {
2672         my $line = <$fh>;
2673         $fh->close;
2674         return undef if !$line;
2675         my @param = split(/\0/, $line);
2676
2677         my $cmd = $param[0];
2678         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2679
2680         for (my $i = 0; $i < scalar (@param); $i++) {
2681             my $p = $param[$i];
2682             next if !$p;
2683             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2684                 my $p = $param[$i+1];
2685                 return 1 if $p && ($p eq $pidfile);
2686                 return undef;
2687             }
2688         }
2689     }
2690     return undef;
2691 }
2692
2693 sub check_running {
2694     my ($vmid, $nocheck, $node) = @_;
2695
2696     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2697
2698     die "unable to find configuration file for VM $vmid - no such machine\n"
2699         if !$nocheck && ! -f $filename;
2700
2701     my $pidfile = pidfile_name($vmid);
2702
2703     if (my $fd = IO::File->new("<$pidfile")) {
2704         my $st = stat($fd);
2705         my $line = <$fd>;
2706         close($fd);
2707
2708         my $mtime = $st->mtime;
2709         if ($mtime > time()) {
2710             warn "file '$filename' modified in future\n";
2711         }
2712
2713         if ($line =~ m/^(\d+)$/) {
2714             my $pid = $1;
2715             if (check_cmdline($pidfile, $pid)) {
2716                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2717                     return $pid;
2718                 }
2719             }
2720         }
2721     }
2722
2723     return undef;
2724 }
2725
2726 sub vzlist {
2727
2728     my $vzlist = config_list();
2729
2730     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2731
2732     while (defined(my $de = $fd->read)) {
2733         next if $de !~ m/^(\d+)\.pid$/;
2734         my $vmid = $1;
2735         next if !defined($vzlist->{$vmid});
2736         if (my $pid = check_running($vmid)) {
2737             $vzlist->{$vmid}->{pid} = $pid;
2738         }
2739     }
2740
2741     return $vzlist;
2742 }
2743
2744 sub disksize {
2745     my ($storecfg, $conf) = @_;
2746
2747     my $bootdisk = $conf->{bootdisk};
2748     return undef if !$bootdisk;
2749     return undef if !is_valid_drivename($bootdisk);
2750
2751     return undef if !$conf->{$bootdisk};
2752
2753     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2754     return undef if !defined($drive);
2755
2756     return undef if drive_is_cdrom($drive);
2757
2758     my $volid = $drive->{file};
2759     return undef if !$volid;
2760
2761     return $drive->{size};
2762 }
2763
2764 my $last_proc_pid_stat;
2765
2766 # get VM status information
2767 # This must be fast and should not block ($full == false)
2768 # We only query KVM using QMP if $full == true (this can be slow)
2769 sub vmstatus {
2770     my ($opt_vmid, $full) = @_;
2771
2772     my $res = {};
2773
2774     my $storecfg = PVE::Storage::config();
2775
2776     my $list = vzlist();
2777     my $defaults = load_defaults();
2778
2779     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2780
2781     my $cpucount = $cpuinfo->{cpus} || 1;
2782
2783     foreach my $vmid (keys %$list) {
2784         next if $opt_vmid && ($vmid ne $opt_vmid);
2785
2786         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2787         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2788
2789         my $d = {};
2790         $d->{pid} = $list->{$vmid}->{pid};
2791
2792         # fixme: better status?
2793         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2794
2795         my $size = disksize($storecfg, $conf);
2796         if (defined($size)) {
2797             $d->{disk} = 0; # no info available
2798             $d->{maxdisk} = $size;
2799         } else {
2800             $d->{disk} = 0;
2801             $d->{maxdisk} = 0;
2802         }
2803
2804         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2805             * ($conf->{cores} || $defaults->{cores});
2806         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2807         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2808
2809         $d->{name} = $conf->{name} || "VM $vmid";
2810         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2811             : $defaults->{memory}*(1024*1024);
2812
2813         if ($conf->{balloon}) {
2814             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2815             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2816                 : $defaults->{shares};
2817         }
2818
2819         $d->{uptime} = 0;
2820         $d->{cpu} = 0;
2821         $d->{mem} = 0;
2822
2823         $d->{netout} = 0;
2824         $d->{netin} = 0;
2825
2826         $d->{diskread} = 0;
2827         $d->{diskwrite} = 0;
2828
2829         $d->{template} = PVE::QemuConfig->is_template($conf);
2830
2831         $d->{serial} = 1 if conf_has_serial($conf);
2832
2833         $res->{$vmid} = $d;
2834     }
2835
2836     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2837     foreach my $dev (keys %$netdev) {
2838         next if $dev !~ m/^tap([1-9]\d*)i/;
2839         my $vmid = $1;
2840         my $d = $res->{$vmid};
2841         next if !$d;
2842
2843         $d->{netout} += $netdev->{$dev}->{receive};
2844         $d->{netin} += $netdev->{$dev}->{transmit};
2845
2846         if ($full) {
2847             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2848             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2849         }
2850
2851     }
2852
2853     my $ctime = gettimeofday;
2854
2855     foreach my $vmid (keys %$list) {
2856
2857         my $d = $res->{$vmid};
2858         my $pid = $d->{pid};
2859         next if !$pid;
2860
2861         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2862         next if !$pstat; # not running
2863
2864         my $used = $pstat->{utime} + $pstat->{stime};
2865
2866         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2867
2868         if ($pstat->{vsize}) {
2869             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2870         }
2871
2872         my $old = $last_proc_pid_stat->{$pid};
2873         if (!$old) {
2874             $last_proc_pid_stat->{$pid} = {
2875                 time => $ctime,
2876                 used => $used,
2877                 cpu => 0,
2878             };
2879             next;
2880         }
2881
2882         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2883
2884         if ($dtime > 1000) {
2885             my $dutime = $used -  $old->{used};
2886
2887             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2888             $last_proc_pid_stat->{$pid} = {
2889                 time => $ctime,
2890                 used => $used,
2891                 cpu => $d->{cpu},
2892             };
2893         } else {
2894             $d->{cpu} = $old->{cpu};
2895         }
2896     }
2897
2898     return $res if !$full;
2899
2900     my $qmpclient = PVE::QMPClient->new();
2901
2902     my $ballooncb = sub {
2903         my ($vmid, $resp) = @_;
2904
2905         my $info = $resp->{'return'};
2906         return if !$info->{max_mem};
2907
2908         my $d = $res->{$vmid};
2909
2910         # use memory assigned to VM
2911         $d->{maxmem} = $info->{max_mem};
2912         $d->{balloon} = $info->{actual};
2913
2914         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2915             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2916             $d->{freemem} = $info->{free_mem};
2917         }
2918
2919         $d->{ballooninfo} = $info;
2920     };
2921
2922     my $blockstatscb = sub {
2923         my ($vmid, $resp) = @_;
2924         my $data = $resp->{'return'} || [];
2925         my $totalrdbytes = 0;
2926         my $totalwrbytes = 0;
2927
2928         for my $blockstat (@$data) {
2929             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2930             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2931
2932             $blockstat->{device} =~ s/drive-//;
2933             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2934         }
2935         $res->{$vmid}->{diskread} = $totalrdbytes;
2936         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2937     };
2938
2939     my $statuscb = sub {
2940         my ($vmid, $resp) = @_;
2941
2942         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2943         # this fails if ballon driver is not loaded, so this must be
2944         # the last commnand (following command are aborted if this fails).
2945         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2946
2947         my $status = 'unknown';
2948         if (!defined($status = $resp->{'return'}->{status})) {
2949             warn "unable to get VM status\n";
2950             return;
2951         }
2952
2953         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2954     };
2955
2956     foreach my $vmid (keys %$list) {
2957         next if $opt_vmid && ($vmid ne $opt_vmid);
2958         next if !$res->{$vmid}->{pid}; # not running
2959         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2960     }
2961
2962     $qmpclient->queue_execute(undef, 2);
2963
2964     foreach my $vmid (keys %$list) {
2965         next if $opt_vmid && ($vmid ne $opt_vmid);
2966         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2967     }
2968
2969     return $res;
2970 }
2971
2972 sub foreach_drive {
2973     my ($conf, $func, @param) = @_;
2974
2975     foreach my $ds (valid_drive_names()) {
2976         next if !defined($conf->{$ds});
2977
2978         my $drive = parse_drive($ds, $conf->{$ds});
2979         next if !$drive;
2980
2981         &$func($ds, $drive, @param);
2982     }
2983 }
2984
2985 sub foreach_volid {
2986     my ($conf, $func, @param) = @_;
2987
2988     my $volhash = {};
2989
2990     my $test_volid = sub {
2991         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
2992
2993         return if !$volid;
2994
2995         $volhash->{$volid}->{cdrom} //= 1;
2996         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
2997
2998         $volhash->{$volid}->{replicate} //= 0;
2999         $volhash->{$volid}->{replicate} = 1 if $replicate;
3000
3001         $volhash->{$volid}->{shared} //= 0;
3002         $volhash->{$volid}->{shared} = 1 if $shared;
3003
3004         $volhash->{$volid}->{referenced_in_config} //= 0;
3005         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3006
3007         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3008             if defined($snapname);
3009     };
3010
3011     foreach_drive($conf, sub {
3012         my ($ds, $drive) = @_;
3013         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3014     });
3015
3016     foreach my $snapname (keys %{$conf->{snapshots}}) {
3017         my $snap = $conf->{snapshots}->{$snapname};
3018         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3019         foreach_drive($snap, sub {
3020             my ($ds, $drive) = @_;
3021             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3022         });
3023     }
3024
3025     foreach my $volid (keys %$volhash) {
3026         &$func($volid, $volhash->{$volid}, @param);
3027     }
3028 }
3029
3030 sub conf_has_serial {
3031     my ($conf) = @_;
3032
3033     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3034         if ($conf->{"serial$i"}) {
3035             return 1;
3036         }
3037     }
3038
3039     return 0;
3040 }
3041
3042 sub vga_conf_has_spice {
3043     my ($vga) = @_;
3044
3045     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
3046
3047     return $1 || 1;
3048 }
3049
3050 sub config_to_command {
3051     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3052
3053     my $cmd = [];
3054     my $globalFlags = [];
3055     my $machineFlags = [];
3056     my $rtcFlags = [];
3057     my $cpuFlags = [];
3058     my $devices = [];
3059     my $pciaddr = '';
3060     my $bridges = {};
3061     my $kvmver = kvm_user_version();
3062     my $vernum = 0; # unknown
3063     my $ostype = $conf->{ostype};
3064     my $winversion = windows_version($ostype);
3065     my $kvm = $conf->{kvm} // 1;
3066
3067     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
3068
3069     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3070         $vernum = $1*1000000+$2*1000;
3071     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3072         $vernum = $1*1000000+$2*1000+$3;
3073     }
3074
3075     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3076
3077     my $have_ovz = -f '/proc/vz/vestat';
3078
3079     my $q35 = machine_type_is_q35($conf);
3080     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3081     my $machine_type = $forcemachine || $conf->{machine};
3082     my $use_old_bios_files = undef;
3083     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3084
3085     my $cpuunits = defined($conf->{cpuunits}) ?
3086             $conf->{cpuunits} : $defaults->{cpuunits};
3087
3088     push @$cmd, '/usr/bin/kvm';
3089
3090     push @$cmd, '-id', $vmid;
3091
3092     my $vmname = $conf->{name} || "vm$vmid";
3093
3094     push @$cmd, '-name', $vmname;
3095
3096     my $use_virtio = 0;
3097
3098     my $qmpsocket = qmp_socket($vmid);
3099     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3100     push @$cmd, '-mon', "chardev=qmp,mode=control";
3101
3102
3103     push @$cmd, '-pidfile' , pidfile_name($vmid);
3104
3105     push @$cmd, '-daemonize';
3106
3107     if ($conf->{smbios1}) {
3108         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3109     }
3110
3111     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3112         die "uefi base image not found\n" if ! -f $OVMF_CODE;
3113
3114         my $path;
3115         my $format;
3116         if (my $efidisk = $conf->{efidisk0}) {
3117             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3118             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3119             $format = $d->{format};
3120             if ($storeid) {
3121                 $path = PVE::Storage::path($storecfg, $d->{file});
3122                 if (!defined($format)) {
3123                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3124                     $format = qemu_img_format($scfg, $volname);
3125                 }
3126             } else {
3127                 $path = $d->{file};
3128                 die "efidisk format must be specified\n"
3129                     if !defined($format);
3130             }
3131         } else {
3132             warn "no efidisk configured! Using temporary efivars disk.\n";
3133             $path = "/tmp/$vmid-ovmf.fd";
3134             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
3135             $format = 'raw';
3136         }
3137
3138         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3139         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3140     }
3141
3142
3143     # add usb controllers
3144     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3145     push @$devices, @usbcontrollers if @usbcontrollers;
3146     my $vga = $conf->{vga};
3147
3148     my $qxlnum = vga_conf_has_spice($vga);
3149     $vga = 'qxl' if $qxlnum;
3150
3151     if (!$vga) {
3152         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3153             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3154         } else {
3155             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
3156         }
3157     }
3158
3159     # enable absolute mouse coordinates (needed by vnc)
3160     my $tablet;
3161     if (defined($conf->{tablet})) {
3162         $tablet = $conf->{tablet};
3163     } else {
3164         $tablet = $defaults->{tablet};
3165         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3166         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3167     }
3168
3169     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3170
3171     my $kvm_off = 0;
3172     my $gpu_passthrough;
3173
3174     # host pci devices
3175     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3176         my $d = parse_hostpci($conf->{"hostpci$i"});
3177         next if !$d;
3178
3179         my $pcie = $d->{pcie};
3180         if($pcie){
3181             die "q35 machine model is not enabled" if !$q35;
3182             $pciaddr = print_pcie_addr("hostpci$i");
3183         }else{
3184             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3185         }
3186
3187         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3188         my $romfile = $d->{romfile};
3189
3190         my $xvga = '';
3191         if ($d->{'x-vga'}) {
3192             $xvga = ',x-vga=on';
3193             $kvm_off = 1;
3194             $vga = 'none';
3195             $gpu_passthrough = 1;
3196
3197             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3198                 $xvga = "";
3199             }
3200         }
3201         my $pcidevices = $d->{pciid};
3202         my $multifunction = 1 if @$pcidevices > 1;
3203
3204         my $j=0;
3205         foreach my $pcidevice (@$pcidevices) {
3206
3207             my $id = "hostpci$i";
3208             $id .= ".$j" if $multifunction;
3209             my $addr = $pciaddr;
3210             $addr .= ".$j" if $multifunction;
3211             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3212
3213             if($j == 0){
3214                 $devicestr .= "$rombar$xvga";
3215                 $devicestr .= ",multifunction=on" if $multifunction;
3216                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3217             }
3218
3219             push @$devices, '-device', $devicestr;
3220             $j++;
3221         }
3222     }
3223
3224     # usb devices
3225     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3226     push @$devices, @usbdevices if @usbdevices;
3227     # serial devices
3228     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3229         if (my $path = $conf->{"serial$i"}) {
3230             if ($path eq 'socket') {
3231                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3232                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3233                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3234             } else {
3235                 die "no such serial device\n" if ! -c $path;
3236                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3237                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3238             }
3239         }
3240     }
3241
3242     # parallel devices
3243     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3244         if (my $path = $conf->{"parallel$i"}) {
3245             die "no such parallel device\n" if ! -c $path;
3246             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3247             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3248             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3249         }
3250     }
3251
3252
3253     my $sockets = 1;
3254     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3255     $sockets = $conf->{sockets} if  $conf->{sockets};
3256
3257     my $cores = $conf->{cores} || 1;
3258
3259     my $maxcpus = $sockets * $cores;
3260
3261     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3262
3263     my $allowed_vcpus = $cpuinfo->{cpus};
3264
3265     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3266         if ($allowed_vcpus < $maxcpus);
3267
3268     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3269
3270         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3271         for (my $i = 2; $i <= $vcpus; $i++)  {
3272             my $cpustr = print_cpu_device($conf,$i);
3273             push @$cmd, '-device', $cpustr;
3274         }
3275
3276     } else {
3277
3278         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3279     }
3280     push @$cmd, '-nodefaults';
3281
3282     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3283
3284     my $bootindex_hash = {};
3285     my $i = 1;
3286     foreach my $o (split(//, $bootorder)) {
3287         $bootindex_hash->{$o} = $i*100;
3288         $i++;
3289     }
3290
3291     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3292
3293     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3294
3295     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3296
3297     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3298
3299     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3300         my $socket = vnc_socket($vmid);
3301         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3302     } else {
3303         push @$cmd, '-nographic';
3304     }
3305
3306     # time drift fix
3307     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3308
3309     my $useLocaltime = $conf->{localtime};
3310
3311     if ($winversion >= 5) { # windows
3312         $useLocaltime = 1 if !defined($conf->{localtime});
3313
3314         # use time drift fix when acpi is enabled
3315         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3316             $tdf = 1 if !defined($conf->{tdf});
3317         }
3318     }
3319
3320     if ($winversion >= 6) {
3321         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3322         push @$cmd, '-no-hpet';
3323     }
3324
3325     push @$rtcFlags, 'driftfix=slew' if $tdf;
3326
3327     if (!$kvm) {
3328         push @$machineFlags, 'accel=tcg';
3329     }
3330
3331     if ($machine_type) {
3332         push @$machineFlags, "type=${machine_type}";
3333     }
3334
3335     if ($conf->{startdate}) {
3336         push @$rtcFlags, "base=$conf->{startdate}";
3337     } elsif ($useLocaltime) {
3338         push @$rtcFlags, 'base=localtime';
3339     }
3340
3341     my $cpu = $kvm ? "kvm64" : "qemu64";
3342     if (my $cputype = $conf->{cpu}) {
3343         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3344             or die "Cannot parse cpu description: $cputype\n";
3345         $cpu = $cpuconf->{cputype};
3346         $kvm_off = 1 if $cpuconf->{hidden};
3347
3348         if (defined(my $flags = $cpuconf->{flags})) {
3349             push @$cpuFlags, split(";", $flags);
3350         }
3351     }
3352
3353     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3354
3355     push @$cpuFlags , '-x2apic'
3356         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3357
3358     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3359
3360     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3361
3362     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3363
3364         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3365         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3366     }
3367
3368     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3369
3370     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3371
3372     push @$cpuFlags, 'kvm=off' if $kvm_off;
3373
3374     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3375         die "internal error"; # should not happen
3376
3377     push @$cpuFlags, "vendor=${cpu_vendor}"
3378         if $cpu_vendor ne 'default';
3379
3380     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3381
3382     push @$cmd, '-cpu', $cpu;
3383
3384     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3385
3386     push @$cmd, '-S' if $conf->{freeze};
3387
3388     push @$cmd, '-k', $conf->{keyboard} if defined($conf->{keyboard});
3389
3390     # enable sound
3391     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3392     #push @$cmd, '-soundhw', 'es1370';
3393     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3394
3395     if($conf->{agent}) {
3396         my $qgasocket = qmp_socket($vmid, 1);
3397         my $pciaddr = print_pci_addr("qga0", $bridges);
3398         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3399         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3400         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3401     }
3402
3403     my $spice_port;
3404
3405     if ($qxlnum) {
3406         if ($qxlnum > 1) {
3407             if ($winversion){
3408                 for(my $i = 1; $i < $qxlnum; $i++){
3409                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3410                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3411                 }
3412             } else {
3413                 # assume other OS works like Linux
3414                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3415                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3416             }
3417         }
3418
3419         my $pciaddr = print_pci_addr("spice", $bridges);
3420
3421         my $nodename = PVE::INotify::nodename();
3422         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3423         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3424         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3425         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3426         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3427
3428         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3429
3430         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3431         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3432         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3433     }
3434
3435     # enable balloon by default, unless explicitly disabled
3436     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3437         $pciaddr = print_pci_addr("balloon0", $bridges);
3438         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3439     }
3440
3441     if ($conf->{watchdog}) {
3442         my $wdopts = parse_watchdog($conf->{watchdog});
3443         $pciaddr = print_pci_addr("watchdog", $bridges);
3444         my $watchdog = $wdopts->{model} || 'i6300esb';
3445         push @$devices, '-device', "$watchdog$pciaddr";
3446         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3447     }
3448
3449     my $vollist = [];
3450     my $scsicontroller = {};
3451     my $ahcicontroller = {};
3452     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3453
3454     # Add iscsi initiator name if available
3455     if (my $initiator = get_initiator_name()) {
3456         push @$devices, '-iscsi', "initiator-name=$initiator";
3457     }
3458
3459     foreach_drive($conf, sub {
3460         my ($ds, $drive) = @_;
3461
3462         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3463             push @$vollist, $drive->{file};
3464         }
3465
3466         # ignore efidisk here, already added in bios/fw handling code above
3467         return if $drive->{interface} eq 'efidisk';
3468
3469         $use_virtio = 1 if $ds =~ m/^virtio/;
3470
3471         if (drive_is_cdrom ($drive)) {
3472             if ($bootindex_hash->{d}) {
3473                 $drive->{bootindex} = $bootindex_hash->{d};
3474                 $bootindex_hash->{d} += 1;
3475             }
3476         } else {
3477             if ($bootindex_hash->{c}) {
3478                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3479                 $bootindex_hash->{c} += 1;
3480             }
3481         }
3482
3483         if($drive->{interface} eq 'virtio'){
3484            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3485         }
3486
3487         if ($drive->{interface} eq 'scsi') {
3488
3489             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3490
3491             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3492             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3493
3494             my $iothread = '';
3495             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3496                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3497                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3498             } elsif ($drive->{iothread}) {
3499                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3500             }
3501
3502             my $queues = '';
3503             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3504                 $queues = ",num_queues=$drive->{queues}";
3505             }
3506
3507             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3508             $scsicontroller->{$controller}=1;
3509         }
3510
3511         if ($drive->{interface} eq 'sata') {
3512            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3513            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3514            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3515            $ahcicontroller->{$controller}=1;
3516         }
3517
3518         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3519         push @$devices, '-drive',$drive_cmd;
3520         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3521     });
3522
3523     for (my $i = 0; $i < $MAX_NETS; $i++) {
3524          next if !$conf->{"net$i"};
3525          my $d = parse_net($conf->{"net$i"});
3526          next if !$d;
3527
3528          $use_virtio = 1 if $d->{model} eq 'virtio';
3529
3530          if ($bootindex_hash->{n}) {
3531             $d->{bootindex} = $bootindex_hash->{n};
3532             $bootindex_hash->{n} += 1;
3533          }
3534
3535          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3536          push @$devices, '-netdev', $netdevfull;
3537
3538          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3539          push @$devices, '-device', $netdevicefull;
3540     }
3541
3542     if (!$q35) {
3543         # add pci bridges
3544         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3545            $bridges->{1} = 1;
3546            $bridges->{2} = 1;
3547         }
3548
3549         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3550
3551         while (my ($k, $v) = each %$bridges) {
3552             $pciaddr = print_pci_addr("pci.$k");
3553             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3554         }
3555     }
3556
3557     # add custom args
3558     if ($conf->{args}) {
3559         my $aa = PVE::Tools::split_args($conf->{args});
3560         push @$cmd, @$aa;
3561     }
3562
3563     push @$cmd, @$devices;
3564     push @$cmd, '-rtc', join(',', @$rtcFlags)
3565         if scalar(@$rtcFlags);
3566     push @$cmd, '-machine', join(',', @$machineFlags)
3567         if scalar(@$machineFlags);
3568     push @$cmd, '-global', join(',', @$globalFlags)
3569         if scalar(@$globalFlags);
3570
3571     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3572 }
3573
3574 sub vnc_socket {
3575     my ($vmid) = @_;
3576     return "${var_run_tmpdir}/$vmid.vnc";
3577 }
3578
3579 sub spice_port {
3580     my ($vmid) = @_;
3581
3582     my $res = vm_mon_cmd($vmid, 'query-spice');
3583
3584     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3585 }
3586
3587 sub qmp_socket {
3588     my ($vmid, $qga) = @_;
3589     my $sockettype = $qga ? 'qga' : 'qmp';
3590     return "${var_run_tmpdir}/$vmid.$sockettype";
3591 }
3592
3593 sub pidfile_name {
3594     my ($vmid) = @_;
3595     return "${var_run_tmpdir}/$vmid.pid";
3596 }
3597
3598 sub vm_devices_list {
3599     my ($vmid) = @_;
3600
3601     my $res = vm_mon_cmd($vmid, 'query-pci');
3602     my $devices = {};
3603     foreach my $pcibus (@$res) {
3604         foreach my $device (@{$pcibus->{devices}}) {
3605             next if !$device->{'qdev_id'};
3606             if ($device->{'pci_bridge'}) {
3607                 $devices->{$device->{'qdev_id'}} = 1;
3608                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3609                     next if !$bridge_device->{'qdev_id'};
3610                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3611                     $devices->{$device->{'qdev_id'}}++;
3612                 }
3613             } else {
3614                 $devices->{$device->{'qdev_id'}} = 1;
3615             }
3616         }
3617     }
3618
3619     my $resblock = vm_mon_cmd($vmid, 'query-block');
3620     foreach my $block (@$resblock) {
3621         if($block->{device} =~ m/^drive-(\S+)/){
3622                 $devices->{$1} = 1;
3623         }
3624     }
3625
3626     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3627     foreach my $mice (@$resmice) {
3628         if ($mice->{name} eq 'QEMU HID Tablet') {
3629             $devices->{tablet} = 1;
3630             last;
3631         }
3632     }
3633
3634     # for usb devices there is no query-usb
3635     # but we can iterate over the entries in
3636     # qom-list path=/machine/peripheral
3637     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3638     foreach my $per (@$resperipheral) {
3639         if ($per->{name} =~ m/^usb\d+$/) {
3640             $devices->{$per->{name}} = 1;
3641         }
3642     }
3643
3644     return $devices;
3645 }
3646
3647 sub vm_deviceplug {
3648     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3649
3650     my $q35 = machine_type_is_q35($conf);
3651
3652     my $devices_list = vm_devices_list($vmid);
3653     return 1 if defined($devices_list->{$deviceid});
3654
3655     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3656
3657     if ($deviceid eq 'tablet') {
3658
3659         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3660
3661     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3662
3663         die "usb hotplug currently not reliable\n";
3664         # since we can't reliably hot unplug all added usb devices
3665         # and usb passthrough disables live migration
3666         # we disable usb hotplugging for now
3667         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3668
3669     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3670
3671         qemu_iothread_add($vmid, $deviceid, $device);
3672
3673         qemu_driveadd($storecfg, $vmid, $device);
3674         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3675
3676         qemu_deviceadd($vmid, $devicefull);
3677         eval { qemu_deviceaddverify($vmid, $deviceid); };
3678         if (my $err = $@) {
3679             eval { qemu_drivedel($vmid, $deviceid); };
3680             warn $@ if $@;
3681             die $err;
3682         }
3683
3684     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3685
3686
3687         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3688         my $pciaddr = print_pci_addr($deviceid);
3689         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3690
3691         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3692
3693         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3694             qemu_iothread_add($vmid, $deviceid, $device);
3695             $devicefull .= ",iothread=iothread-$deviceid";
3696         }
3697
3698         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3699             $devicefull .= ",num_queues=$device->{queues}";
3700         }
3701
3702         qemu_deviceadd($vmid, $devicefull);
3703         qemu_deviceaddverify($vmid, $deviceid);
3704
3705     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3706
3707         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3708         qemu_driveadd($storecfg, $vmid, $device);
3709
3710         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3711         eval { qemu_deviceadd($vmid, $devicefull); };
3712         if (my $err = $@) {
3713             eval { qemu_drivedel($vmid, $deviceid); };
3714             warn $@ if $@;
3715             die $err;
3716         }
3717
3718     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3719
3720         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3721
3722         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3723         my $use_old_bios_files = undef;
3724         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3725
3726         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3727         qemu_deviceadd($vmid, $netdevicefull);
3728         eval { qemu_deviceaddverify($vmid, $deviceid); };
3729         if (my $err = $@) {
3730             eval { qemu_netdevdel($vmid, $deviceid); };
3731             warn $@ if $@;
3732             die $err;
3733         }
3734
3735     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3736
3737         my $bridgeid = $2;
3738         my $pciaddr = print_pci_addr($deviceid);
3739         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3740
3741         qemu_deviceadd($vmid, $devicefull);
3742         qemu_deviceaddverify($vmid, $deviceid);
3743
3744     } else {
3745         die "can't hotplug device '$deviceid'\n";
3746     }
3747
3748     return 1;
3749 }
3750
3751 # fixme: this should raise exceptions on error!
3752 sub vm_deviceunplug {
3753     my ($vmid, $conf, $deviceid) = @_;
3754
3755     my $devices_list = vm_devices_list($vmid);
3756     return 1 if !defined($devices_list->{$deviceid});
3757
3758     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3759
3760     if ($deviceid eq 'tablet') {
3761
3762         qemu_devicedel($vmid, $deviceid);
3763
3764     } elsif ($deviceid =~ m/^usb\d+$/) {
3765
3766         die "usb hotplug currently not reliable\n";
3767         # when unplugging usb devices this way,
3768         # there may be remaining usb controllers/hubs
3769         # so we disable it for now
3770         qemu_devicedel($vmid, $deviceid);
3771         qemu_devicedelverify($vmid, $deviceid);
3772
3773     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3774
3775         qemu_devicedel($vmid, $deviceid);
3776         qemu_devicedelverify($vmid, $deviceid);
3777         qemu_drivedel($vmid, $deviceid);
3778         qemu_iothread_del($conf, $vmid, $deviceid);
3779
3780     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3781
3782         qemu_devicedel($vmid, $deviceid);
3783         qemu_devicedelverify($vmid, $deviceid);
3784         qemu_iothread_del($conf, $vmid, $deviceid);
3785
3786     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3787
3788         qemu_devicedel($vmid, $deviceid);
3789         qemu_drivedel($vmid, $deviceid);
3790         qemu_deletescsihw($conf, $vmid, $deviceid);
3791
3792     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3793
3794         qemu_devicedel($vmid, $deviceid);
3795         qemu_devicedelverify($vmid, $deviceid);
3796         qemu_netdevdel($vmid, $deviceid);
3797
3798     } else {
3799         die "can't unplug device '$deviceid'\n";
3800     }
3801
3802     return 1;
3803 }
3804
3805 sub qemu_deviceadd {
3806     my ($vmid, $devicefull) = @_;
3807
3808     $devicefull = "driver=".$devicefull;
3809     my %options =  split(/[=,]/, $devicefull);
3810
3811     vm_mon_cmd($vmid, "device_add" , %options);
3812 }
3813
3814 sub qemu_devicedel {
3815     my ($vmid, $deviceid) = @_;
3816
3817     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3818 }
3819
3820 sub qemu_iothread_add {
3821     my($vmid, $deviceid, $device) = @_;
3822
3823     if ($device->{iothread}) {
3824         my $iothreads = vm_iothreads_list($vmid);
3825         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3826     }
3827 }
3828
3829 sub qemu_iothread_del {
3830     my($conf, $vmid, $deviceid) = @_;
3831
3832     my $device = parse_drive($deviceid, $conf->{$deviceid});
3833     if ($device->{iothread}) {
3834         my $iothreads = vm_iothreads_list($vmid);
3835         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3836     }
3837 }
3838
3839 sub qemu_objectadd {
3840     my($vmid, $objectid, $qomtype) = @_;
3841
3842     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3843
3844     return 1;
3845 }
3846
3847 sub qemu_objectdel {
3848     my($vmid, $objectid) = @_;
3849
3850     vm_mon_cmd($vmid, "object-del", id => $objectid);
3851
3852     return 1;
3853 }
3854
3855 sub qemu_driveadd {
3856     my ($storecfg, $vmid, $device) = @_;
3857
3858     my $drive = print_drive_full($storecfg, $vmid, $device);
3859     $drive =~ s/\\/\\\\/g;
3860     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3861
3862     # If the command succeeds qemu prints: "OK"
3863     return 1 if $ret =~ m/OK/s;
3864
3865     die "adding drive failed: $ret\n";
3866 }
3867
3868 sub qemu_drivedel {
3869     my($vmid, $deviceid) = @_;
3870
3871     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3872     $ret =~ s/^\s+//;
3873
3874     return 1 if $ret eq "";
3875
3876     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3877     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3878
3879     die "deleting drive $deviceid failed : $ret\n";
3880 }
3881
3882 sub qemu_deviceaddverify {
3883     my ($vmid, $deviceid) = @_;
3884
3885     for (my $i = 0; $i <= 5; $i++) {
3886          my $devices_list = vm_devices_list($vmid);
3887          return 1 if defined($devices_list->{$deviceid});
3888          sleep 1;
3889     }
3890
3891     die "error on hotplug device '$deviceid'\n";
3892 }
3893
3894
3895 sub qemu_devicedelverify {
3896     my ($vmid, $deviceid) = @_;
3897
3898     # need to verify that the device is correctly removed as device_del
3899     # is async and empty return is not reliable
3900
3901     for (my $i = 0; $i <= 5; $i++) {
3902          my $devices_list = vm_devices_list($vmid);
3903          return 1 if !defined($devices_list->{$deviceid});
3904          sleep 1;
3905     }
3906
3907     die "error on hot-unplugging device '$deviceid'\n";
3908 }
3909
3910 sub qemu_findorcreatescsihw {
3911     my ($storecfg, $conf, $vmid, $device) = @_;
3912
3913     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3914
3915     my $scsihwid="$controller_prefix$controller";
3916     my $devices_list = vm_devices_list($vmid);
3917
3918     if(!defined($devices_list->{$scsihwid})) {
3919         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3920     }
3921
3922     return 1;
3923 }
3924
3925 sub qemu_deletescsihw {
3926     my ($conf, $vmid, $opt) = @_;
3927
3928     my $device = parse_drive($opt, $conf->{$opt});
3929
3930     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3931         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3932         return 1;
3933     }
3934
3935     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3936
3937     my $devices_list = vm_devices_list($vmid);
3938     foreach my $opt (keys %{$devices_list}) {
3939         if (PVE::QemuServer::is_valid_drivename($opt)) {
3940             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3941             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3942                 return 1;
3943             }
3944         }
3945     }
3946
3947     my $scsihwid="scsihw$controller";
3948
3949     vm_deviceunplug($vmid, $conf, $scsihwid);
3950
3951     return 1;
3952 }
3953
3954 sub qemu_add_pci_bridge {
3955     my ($storecfg, $conf, $vmid, $device) = @_;
3956
3957     my $bridges = {};
3958
3959     my $bridgeid;
3960
3961     print_pci_addr($device, $bridges);
3962
3963     while (my ($k, $v) = each %$bridges) {
3964         $bridgeid = $k;
3965     }
3966     return 1 if !defined($bridgeid) || $bridgeid < 1;
3967
3968     my $bridge = "pci.$bridgeid";
3969     my $devices_list = vm_devices_list($vmid);
3970
3971     if (!defined($devices_list->{$bridge})) {
3972         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3973     }
3974
3975     return 1;
3976 }
3977
3978 sub qemu_set_link_status {
3979     my ($vmid, $device, $up) = @_;
3980
3981     vm_mon_cmd($vmid, "set_link", name => $device,
3982                up => $up ? JSON::true : JSON::false);
3983 }
3984
3985 sub qemu_netdevadd {
3986     my ($vmid, $conf, $device, $deviceid) = @_;
3987
3988     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3989     my %options =  split(/[=,]/, $netdev);
3990
3991     vm_mon_cmd($vmid, "netdev_add",  %options);
3992     return 1;
3993 }
3994
3995 sub qemu_netdevdel {
3996     my ($vmid, $deviceid) = @_;
3997
3998     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
3999 }
4000
4001 sub qemu_usb_hotplug {
4002     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
4003
4004     return if !$device;
4005
4006     # remove the old one first
4007     vm_deviceunplug($vmid, $conf, $deviceid);
4008
4009     # check if xhci controller is necessary and available
4010     if ($device->{usb3}) {
4011
4012         my $devicelist = vm_devices_list($vmid);
4013
4014         if (!$devicelist->{xhci}) {
4015             my $pciaddr = print_pci_addr("xhci");
4016             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4017         }
4018     }
4019     my $d = parse_usb_device($device->{host});
4020     $d->{usb3} = $device->{usb3};
4021
4022     # add the new one
4023     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4024 }
4025
4026 sub qemu_cpu_hotplug {
4027     my ($vmid, $conf, $vcpus) = @_;
4028
4029     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4030
4031     my $sockets = 1;
4032     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4033     $sockets = $conf->{sockets} if  $conf->{sockets};
4034     my $cores = $conf->{cores} || 1;
4035     my $maxcpus = $sockets * $cores;
4036
4037     $vcpus = $maxcpus if !$vcpus;
4038
4039     die "you can't add more vcpus than maxcpus\n"
4040         if $vcpus > $maxcpus;
4041
4042     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4043
4044     if ($vcpus < $currentvcpus) {
4045
4046         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4047
4048             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4049                 qemu_devicedel($vmid, "cpu$i");
4050                 my $retry = 0;
4051                 my $currentrunningvcpus = undef;
4052                 while (1) {
4053                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4054                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4055                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4056                     $retry++;
4057                     sleep 1;
4058                 }
4059                 #update conf after each succesfull cpu unplug
4060                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4061                 PVE::QemuConfig->write_config($vmid, $conf);
4062             }
4063         } else {
4064             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4065         }
4066
4067         return;
4068     }
4069
4070     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4071     die "vcpus in running vm does not match its configuration\n"
4072         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4073
4074     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4075
4076         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4077             my $cpustr = print_cpu_device($conf, $i);
4078             qemu_deviceadd($vmid, $cpustr);
4079
4080             my $retry = 0;
4081             my $currentrunningvcpus = undef;
4082             while (1) {
4083                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4084                 last if scalar(@{$currentrunningvcpus}) == $i;
4085                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4086                 sleep 1;
4087                 $retry++;
4088             }
4089             #update conf after each succesfull cpu hotplug
4090             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4091             PVE::QemuConfig->write_config($vmid, $conf);
4092         }
4093     } else {
4094
4095         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4096             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4097         }
4098     }
4099 }
4100
4101 sub qemu_block_set_io_throttle {
4102     my ($vmid, $deviceid,
4103         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4104         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4105         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4106         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4107
4108     return if !check_running($vmid) ;
4109
4110     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4111         bps => int($bps),
4112         bps_rd => int($bps_rd),
4113         bps_wr => int($bps_wr),
4114         iops => int($iops),
4115         iops_rd => int($iops_rd),
4116         iops_wr => int($iops_wr),
4117         bps_max => int($bps_max),
4118         bps_rd_max => int($bps_rd_max),
4119         bps_wr_max => int($bps_wr_max),
4120         iops_max => int($iops_max),
4121         iops_rd_max => int($iops_rd_max),
4122         iops_wr_max => int($iops_wr_max),
4123         bps_max_length => int($bps_max_length),
4124         bps_rd_max_length => int($bps_rd_max_length),
4125         bps_wr_max_length => int($bps_wr_max_length),
4126         iops_max_length => int($iops_max_length),
4127         iops_rd_max_length => int($iops_rd_max_length),
4128         iops_wr_max_length => int($iops_wr_max_length),
4129     );
4130
4131 }
4132
4133 # old code, only used to shutdown old VM after update
4134 sub __read_avail {
4135     my ($fh, $timeout) = @_;
4136
4137     my $sel = new IO::Select;
4138     $sel->add($fh);
4139
4140     my $res = '';
4141     my $buf;
4142
4143     my @ready;
4144     while (scalar (@ready = $sel->can_read($timeout))) {
4145         my $count;
4146         if ($count = $fh->sysread($buf, 8192)) {
4147             if ($buf =~ /^(.*)\(qemu\) $/s) {
4148                 $res .= $1;
4149                 last;
4150             } else {
4151                 $res .= $buf;
4152             }
4153         } else {
4154             if (!defined($count)) {
4155                 die "$!\n";
4156             }
4157             last;
4158         }
4159     }
4160
4161     die "monitor read timeout\n" if !scalar(@ready);
4162
4163     return $res;
4164 }
4165
4166 sub qemu_block_resize {
4167     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4168
4169     my $running = check_running($vmid);
4170
4171     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4172
4173     return if !$running;
4174
4175     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4176
4177 }
4178
4179 sub qemu_volume_snapshot {
4180     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4181
4182     my $running = check_running($vmid);
4183
4184     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4185         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4186     } else {
4187         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4188     }
4189 }
4190
4191 sub qemu_volume_snapshot_delete {
4192     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4193
4194     my $running = check_running($vmid);
4195
4196     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4197         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4198     } else {
4199         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4200     }
4201 }
4202
4203 sub set_migration_caps {
4204     my ($vmid) = @_;
4205
4206     my $cap_ref = [];
4207
4208     my $enabled_cap = {
4209         "auto-converge" => 1,
4210         "xbzrle" => 1,
4211         "x-rdma-pin-all" => 0,
4212         "zero-blocks" => 0,
4213         "compress" => 0
4214     };
4215
4216     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4217
4218     for my $supported_capability (@$supported_capabilities) {
4219         push @$cap_ref, {
4220             capability => $supported_capability->{capability},
4221             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4222         };
4223     }
4224
4225     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4226 }
4227
4228 my $fast_plug_option = {
4229     'lock' => 1,
4230     'name' => 1,
4231     'onboot' => 1,
4232     'shares' => 1,
4233     'startup' => 1,
4234     'description' => 1,
4235     'protection' => 1,
4236     'vmstatestorage' => 1,
4237 };
4238
4239 # hotplug changes in [PENDING]
4240 # $selection hash can be used to only apply specified options, for
4241 # example: { cores => 1 } (only apply changed 'cores')
4242 # $errors ref is used to return error messages
4243 sub vmconfig_hotplug_pending {
4244     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4245
4246     my $defaults = load_defaults();
4247
4248     # commit values which do not have any impact on running VM first
4249     # Note: those option cannot raise errors, we we do not care about
4250     # $selection and always apply them.
4251
4252     my $add_error = sub {
4253         my ($opt, $msg) = @_;
4254         $errors->{$opt} = "hotplug problem - $msg";
4255     };
4256
4257     my $changes = 0;
4258     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4259         if ($fast_plug_option->{$opt}) {
4260             $conf->{$opt} = $conf->{pending}->{$opt};
4261             delete $conf->{pending}->{$opt};
4262             $changes = 1;
4263         }
4264     }
4265
4266     if ($changes) {
4267         PVE::QemuConfig->write_config($vmid, $conf);
4268         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4269     }
4270
4271     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4272
4273     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4274     while (my ($opt, $force) = each %$pending_delete_hash) {
4275         next if $selection && !$selection->{$opt};
4276         eval {
4277             if ($opt eq 'hotplug') {
4278                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4279             } elsif ($opt eq 'tablet') {
4280                 die "skip\n" if !$hotplug_features->{usb};
4281                 if ($defaults->{tablet}) {
4282                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4283                 } else {
4284                     vm_deviceunplug($vmid, $conf, $opt);
4285                 }
4286             } elsif ($opt =~ m/^usb\d+/) {
4287                 die "skip\n";
4288                 # since we cannot reliably hot unplug usb devices
4289                 # we are disabling it
4290                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4291                 vm_deviceunplug($vmid, $conf, $opt);
4292             } elsif ($opt eq 'vcpus') {
4293                 die "skip\n" if !$hotplug_features->{cpu};
4294                 qemu_cpu_hotplug($vmid, $conf, undef);
4295             } elsif ($opt eq 'balloon') {
4296                 # enable balloon device is not hotpluggable
4297                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4298             } elsif ($fast_plug_option->{$opt}) {
4299                 # do nothing
4300             } elsif ($opt =~ m/^net(\d+)$/) {
4301                 die "skip\n" if !$hotplug_features->{network};
4302                 vm_deviceunplug($vmid, $conf, $opt);
4303             } elsif (is_valid_drivename($opt)) {
4304                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4305                 vm_deviceunplug($vmid, $conf, $opt);
4306                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4307             } elsif ($opt =~ m/^memory$/) {
4308                 die "skip\n" if !$hotplug_features->{memory};
4309                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4310             } elsif ($opt eq 'cpuunits') {
4311                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4312             } elsif ($opt eq 'cpulimit') {
4313                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4314             } else {
4315                 die "skip\n";
4316             }
4317         };
4318         if (my $err = $@) {
4319             &$add_error($opt, $err) if $err ne "skip\n";
4320         } else {
4321             # save new config if hotplug was successful
4322             delete $conf->{$opt};
4323             vmconfig_undelete_pending_option($conf, $opt);
4324             PVE::QemuConfig->write_config($vmid, $conf);
4325             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4326         }
4327     }
4328
4329     my $apply_pending_cloudinit;
4330     $apply_pending_cloudinit = sub {
4331         my ($key, $value) = @_;
4332         $apply_pending_cloudinit = sub {}; # once is enough
4333
4334         my @cloudinit_opts = keys %$confdesc_cloudinit;
4335         foreach my $opt (keys %{$conf->{pending}}) {
4336             next if !grep { $_ eq $opt } @cloudinit_opts;
4337             $conf->{$opt} = delete $conf->{pending}->{$opt};
4338         }
4339
4340         my $new_conf = { %$conf };
4341         $new_conf->{$key} = $value;
4342         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4343     };
4344
4345     foreach my $opt (keys %{$conf->{pending}}) {
4346         next if $selection && !$selection->{$opt};
4347         my $value = $conf->{pending}->{$opt};
4348         eval {
4349             if ($opt eq 'hotplug') {
4350                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4351             } elsif ($opt eq 'tablet') {
4352                 die "skip\n" if !$hotplug_features->{usb};
4353                 if ($value == 1) {
4354                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4355                 } elsif ($value == 0) {
4356                     vm_deviceunplug($vmid, $conf, $opt);
4357                 }
4358             } elsif ($opt =~ m/^usb\d+$/) {
4359                 die "skip\n";
4360                 # since we cannot reliably hot unplug usb devices
4361                 # we are disabling it
4362                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4363                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4364                 die "skip\n" if !$d;
4365                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4366             } elsif ($opt eq 'vcpus') {
4367                 die "skip\n" if !$hotplug_features->{cpu};
4368                 qemu_cpu_hotplug($vmid, $conf, $value);
4369             } elsif ($opt eq 'balloon') {
4370                 # enable/disable balloning device is not hotpluggable
4371                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4372                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4373                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4374
4375                 # allow manual ballooning if shares is set to zero
4376                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4377                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4378                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4379                 }
4380             } elsif ($opt =~ m/^net(\d+)$/) {
4381                 # some changes can be done without hotplug
4382                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4383                                     $vmid, $opt, $value);
4384             } elsif (is_valid_drivename($opt)) {
4385                 # some changes can be done without hotplug
4386                 my $drive = parse_drive($opt, $value);
4387                 if (drive_is_cloudinit($drive)) {
4388                     &$apply_pending_cloudinit($opt, $value);
4389                 }
4390                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4391                                      $vmid, $opt, $value, 1);
4392             } elsif ($opt =~ m/^memory$/) { #dimms
4393                 die "skip\n" if !$hotplug_features->{memory};
4394                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4395             } elsif ($opt eq 'cpuunits') {
4396                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4397             } elsif ($opt eq 'cpulimit') {
4398                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4399                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4400             } else {
4401                 die "skip\n";  # skip non-hot-pluggable options
4402             }
4403         };
4404         if (my $err = $@) {
4405             &$add_error($opt, $err) if $err ne "skip\n";
4406         } else {
4407             # save new config if hotplug was successful
4408             $conf->{$opt} = $value;
4409             delete $conf->{pending}->{$opt};
4410             PVE::QemuConfig->write_config($vmid, $conf);
4411             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4412         }
4413     }
4414 }
4415
4416 sub try_deallocate_drive {
4417     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4418
4419     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4420         my $volid = $drive->{file};
4421         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4422             my $sid = PVE::Storage::parse_volume_id($volid);
4423             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4424
4425             # check if the disk is really unused
4426             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4427                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4428             PVE::Storage::vdisk_free($storecfg, $volid);
4429             return 1;
4430         } else {
4431             # If vm is not owner of this disk remove from config
4432             return 1;
4433         }
4434     }
4435
4436     return undef;
4437 }
4438
4439 sub vmconfig_delete_or_detach_drive {
4440     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4441
4442     my $drive = parse_drive($opt, $conf->{$opt});
4443
4444     my $rpcenv = PVE::RPCEnvironment::get();
4445     my $authuser = $rpcenv->get_user();
4446
4447     if ($force) {
4448         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4449         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4450     } else {
4451         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4452     }
4453 }
4454
4455 sub vmconfig_apply_pending {
4456     my ($vmid, $conf, $storecfg) = @_;
4457
4458     # cold plug
4459
4460     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4461     while (my ($opt, $force) = each %$pending_delete_hash) {
4462         die "internal error" if $opt =~ m/^unused/;
4463         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4464         if (!defined($conf->{$opt})) {
4465             vmconfig_undelete_pending_option($conf, $opt);
4466             PVE::QemuConfig->write_config($vmid, $conf);
4467         } elsif (is_valid_drivename($opt)) {
4468             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4469             vmconfig_undelete_pending_option($conf, $opt);
4470             delete $conf->{$opt};
4471             PVE::QemuConfig->write_config($vmid, $conf);
4472         } else {
4473             vmconfig_undelete_pending_option($conf, $opt);
4474             delete $conf->{$opt};
4475             PVE::QemuConfig->write_config($vmid, $conf);
4476         }
4477     }
4478
4479     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4480
4481     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4482         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4483
4484         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4485             # skip if nothing changed
4486         } elsif (is_valid_drivename($opt)) {
4487             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4488                 if defined($conf->{$opt});
4489             $conf->{$opt} = $conf->{pending}->{$opt};
4490         } else {
4491             $conf->{$opt} = $conf->{pending}->{$opt};
4492         }
4493
4494         delete $conf->{pending}->{$opt};
4495         PVE::QemuConfig->write_config($vmid, $conf);
4496     }
4497 }
4498
4499 my $safe_num_ne = sub {
4500     my ($a, $b) = @_;
4501
4502     return 0 if !defined($a) && !defined($b);
4503     return 1 if !defined($a);
4504     return 1 if !defined($b);
4505
4506     return $a != $b;
4507 };
4508
4509 my $safe_string_ne = sub {
4510     my ($a, $b) = @_;
4511
4512     return 0 if !defined($a) && !defined($b);
4513     return 1 if !defined($a);
4514     return 1 if !defined($b);
4515
4516     return $a ne $b;
4517 };
4518
4519 sub vmconfig_update_net {
4520     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4521
4522     my $newnet = parse_net($value);
4523
4524     if ($conf->{$opt}) {
4525         my $oldnet = parse_net($conf->{$opt});
4526
4527         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4528             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4529             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4530             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4531
4532             # for non online change, we try to hot-unplug
4533             die "skip\n" if !$hotplug;
4534             vm_deviceunplug($vmid, $conf, $opt);
4535         } else {
4536
4537             die "internal error" if $opt !~ m/net(\d+)/;
4538             my $iface = "tap${vmid}i$1";
4539
4540             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4541                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4542                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4543                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4544                 PVE::Network::tap_unplug($iface);
4545                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4546             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4547                 # Rate can be applied on its own but any change above needs to
4548                 # include the rate in tap_plug since OVS resets everything.
4549                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4550             }
4551
4552             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4553                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4554             }
4555
4556             return 1;
4557         }
4558     }
4559
4560     if ($hotplug) {
4561         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4562     } else {
4563         die "skip\n";
4564     }
4565 }
4566
4567 sub vmconfig_update_disk {
4568     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4569
4570     # fixme: do we need force?
4571
4572     my $drive = parse_drive($opt, $value);
4573
4574     if ($conf->{$opt}) {
4575
4576         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4577
4578             my $media = $drive->{media} || 'disk';
4579             my $oldmedia = $old_drive->{media} || 'disk';
4580             die "unable to change media type\n" if $media ne $oldmedia;
4581
4582             if (!drive_is_cdrom($old_drive)) {
4583
4584                 if ($drive->{file} ne $old_drive->{file}) {
4585
4586                     die "skip\n" if !$hotplug;
4587
4588                     # unplug and register as unused
4589                     vm_deviceunplug($vmid, $conf, $opt);
4590                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4591
4592                 } else {
4593                     # update existing disk
4594
4595                     # skip non hotpluggable value
4596                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4597                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4598                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4599                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4600                         die "skip\n";
4601                     }
4602
4603                     # apply throttle
4604                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4605                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4606                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4607                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4608                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4609                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4610                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4611                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4612                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4613                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4614                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4615                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4616                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4617                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4618                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4619                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4620                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4621                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4622
4623                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4624                                                    ($drive->{mbps} || 0)*1024*1024,
4625                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4626                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4627                                                    $drive->{iops} || 0,
4628                                                    $drive->{iops_rd} || 0,
4629                                                    $drive->{iops_wr} || 0,
4630                                                    ($drive->{mbps_max} || 0)*1024*1024,
4631                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4632                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4633                                                    $drive->{iops_max} || 0,
4634                                                    $drive->{iops_rd_max} || 0,
4635                                                    $drive->{iops_wr_max} || 0,
4636                                                    $drive->{bps_max_length} || 1,
4637                                                    $drive->{bps_rd_max_length} || 1,
4638                                                    $drive->{bps_wr_max_length} || 1,
4639                                                    $drive->{iops_max_length} || 1,
4640                                                    $drive->{iops_rd_max_length} || 1,
4641                                                    $drive->{iops_wr_max_length} || 1);
4642
4643                     }
4644
4645                     return 1;
4646                 }
4647
4648             } else { # cdrom
4649
4650                 if ($drive->{file} eq 'none') {
4651                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4652                     if (drive_is_cloudinit($old_drive)) {
4653                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4654                     }
4655                 } else {
4656                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4657                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4658                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4659                 }
4660
4661                 return 1;
4662             }
4663         }
4664     }
4665
4666     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4667     # hotplug new disks
4668     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4669     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4670 }
4671
4672 sub vm_start {
4673     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4674         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4675
4676     PVE::QemuConfig->lock_config($vmid, sub {
4677         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4678
4679         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4680
4681         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4682
4683         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4684
4685         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4686             vmconfig_apply_pending($vmid, $conf, $storecfg);
4687             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4688         }
4689
4690         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4691
4692         my $defaults = load_defaults();
4693
4694         # set environment variable useful inside network script
4695         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4696
4697         my $local_volumes = {};
4698
4699         if ($targetstorage) {
4700             foreach_drive($conf, sub {
4701                 my ($ds, $drive) = @_;
4702
4703                 return if drive_is_cdrom($drive);
4704
4705                 my $volid = $drive->{file};
4706
4707                 return if !$volid;
4708
4709                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4710
4711                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4712                 return if $scfg->{shared};
4713                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4714             });
4715
4716             my $format = undef;
4717
4718             foreach my $opt (sort keys %$local_volumes) {
4719
4720                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4721                 my $drive = parse_drive($opt, $conf->{$opt});
4722
4723                 #if remote storage is specified, use default format
4724                 if ($targetstorage && $targetstorage ne "1") {
4725                     $storeid = $targetstorage;
4726                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4727                     $format = $defFormat;
4728                 } else {
4729                     #else we use same format than original
4730                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4731                     $format = qemu_img_format($scfg, $volid);
4732                 }
4733
4734                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4735                 my $newdrive = $drive;
4736                 $newdrive->{format} = $format;
4737                 $newdrive->{file} = $newvolid;
4738                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4739                 $local_volumes->{$opt} = $drivestr;
4740                 #pass drive to conf for command line
4741                 $conf->{$opt} = $drivestr;
4742             }
4743         }
4744
4745         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4746
4747         my $migrate_port = 0;
4748         my $migrate_uri;
4749         if ($statefile) {
4750             if ($statefile eq 'tcp') {
4751                 my $localip = "localhost";
4752                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4753                 my $nodename = PVE::INotify::nodename();
4754
4755                 if (!defined($migration_type)) {
4756                     if (defined($datacenterconf->{migration}->{type})) {
4757                         $migration_type = $datacenterconf->{migration}->{type};
4758                     } else {
4759                         $migration_type = 'secure';
4760                     }
4761                 }
4762
4763                 if ($migration_type eq 'insecure') {
4764                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4765                     if ($migrate_network_addr) {
4766                         $localip = $migrate_network_addr;
4767                     } else {
4768                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4769                     }
4770
4771                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4772                 }
4773
4774                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4775                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4776                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4777                 push @$cmd, '-incoming', $migrate_uri;
4778                 push @$cmd, '-S';
4779
4780             } elsif ($statefile eq 'unix') {
4781                 # should be default for secure migrations as a ssh TCP forward
4782                 # tunnel is not deterministic reliable ready and fails regurarly
4783                 # to set up in time, so use UNIX socket forwards
4784                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4785                 unlink $socket_addr;
4786
4787                 $migrate_uri = "unix:$socket_addr";
4788
4789                 push @$cmd, '-incoming', $migrate_uri;
4790                 push @$cmd, '-S';
4791
4792             } else {
4793                 push @$cmd, '-loadstate', $statefile;
4794             }
4795         } elsif ($paused) {
4796             push @$cmd, '-S';
4797         }
4798
4799         # host pci devices
4800         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4801           my $d = parse_hostpci($conf->{"hostpci$i"});
4802           next if !$d;
4803           my $pcidevices = $d->{pciid};
4804           foreach my $pcidevice (@$pcidevices) {
4805                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4806
4807                 my $info = pci_device_info("0000:$pciid");
4808                 die "IOMMU not present\n" if !check_iommu_support();
4809                 die "no pci device info for device '$pciid'\n" if !$info;
4810                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4811                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4812           }
4813         }
4814
4815         PVE::Storage::activate_volumes($storecfg, $vollist);
4816
4817         if (!check_running($vmid, 1)) {
4818             eval {
4819                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4820                     outfunc => sub {}, errfunc => sub {});
4821             };
4822         }
4823
4824         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4825                                                   : $defaults->{cpuunits};
4826
4827         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4828         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4829
4830         my %properties = (
4831             Slice => 'qemu.slice',
4832             KillMode => 'none',
4833             CPUShares => $cpuunits
4834         );
4835
4836         if (my $cpulimit = $conf->{cpulimit}) {
4837             $properties{CPUQuota} = int($cpulimit * 100);
4838         }
4839         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4840
4841         if ($conf->{hugepages}) {
4842
4843             my $code = sub {
4844                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4845                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4846
4847                 PVE::QemuServer::Memory::hugepages_mount();
4848                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4849
4850                 eval  {
4851                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4852                     run_command($cmd, %run_params);
4853                 };
4854
4855                 if (my $err = $@) {
4856                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4857                     die $err;
4858                 }
4859
4860                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4861             };
4862             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4863
4864         } else {
4865             eval  {
4866                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4867                 run_command($cmd, %run_params);
4868             };
4869         }
4870
4871         if (my $err = $@) {
4872             # deactivate volumes if start fails
4873             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4874             die "start failed: $err";
4875         }
4876
4877         print "migration listens on $migrate_uri\n" if $migrate_uri;
4878
4879         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4880             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4881             warn $@ if $@;
4882         }
4883
4884         #start nbd server for storage migration
4885         if ($targetstorage) {
4886             my $nodename = PVE::INotify::nodename();
4887             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4888             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4889             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4890             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4891
4892             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4893
4894             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4895
4896             foreach my $opt (sort keys %$local_volumes) {
4897                 my $volid = $local_volumes->{$opt};
4898                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4899                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4900                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4901             }
4902         }
4903
4904         if ($migratedfrom) {
4905             eval {
4906                 set_migration_caps($vmid);
4907             };
4908             warn $@ if $@;
4909
4910             if ($spice_port) {
4911                 print "spice listens on port $spice_port\n";
4912                 if ($spice_ticket) {
4913                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4914                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4915                 }
4916             }
4917
4918         } else {
4919             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4920                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4921                     if $conf->{balloon};
4922             }
4923
4924             foreach my $opt (keys %$conf) {
4925                 next if $opt !~  m/^net\d+$/;
4926                 my $nicconf = parse_net($conf->{$opt});
4927                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4928             }
4929         }
4930
4931         vm_mon_cmd_nocheck($vmid, 'qom-set',
4932                     path => "machine/peripheral/balloon0",
4933                     property => "guest-stats-polling-interval",
4934                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4935
4936     });
4937 }
4938
4939 sub vm_mon_cmd {
4940     my ($vmid, $execute, %params) = @_;
4941
4942     my $cmd = { execute => $execute, arguments => \%params };
4943     vm_qmp_command($vmid, $cmd);
4944 }
4945
4946 sub vm_mon_cmd_nocheck {
4947     my ($vmid, $execute, %params) = @_;
4948
4949     my $cmd = { execute => $execute, arguments => \%params };
4950     vm_qmp_command($vmid, $cmd, 1);
4951 }
4952
4953 sub vm_qmp_command {
4954     my ($vmid, $cmd, $nocheck) = @_;
4955
4956     my $res;
4957
4958     my $timeout;
4959     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4960         $timeout = $cmd->{arguments}->{timeout};
4961         delete $cmd->{arguments}->{timeout};
4962     }
4963
4964     eval {
4965         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4966         my $sname = qmp_socket($vmid);
4967         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4968             my $qmpclient = PVE::QMPClient->new();
4969
4970             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4971         } else {
4972             die "unable to open monitor socket\n";
4973         }
4974     };
4975     if (my $err = $@) {
4976         syslog("err", "VM $vmid qmp command failed - $err");
4977         die $err;
4978     }
4979
4980     return $res;
4981 }
4982
4983 sub vm_human_monitor_command {
4984     my ($vmid, $cmdline) = @_;
4985
4986     my $res;
4987
4988     my $cmd = {
4989         execute => 'human-monitor-command',
4990         arguments => { 'command-line' => $cmdline},
4991     };
4992
4993     return vm_qmp_command($vmid, $cmd);
4994 }
4995
4996 sub vm_commandline {
4997     my ($storecfg, $vmid) = @_;
4998
4999     my $conf = PVE::QemuConfig->load_config($vmid);
5000
5001     my $defaults = load_defaults();
5002
5003     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5004
5005     return PVE::Tools::cmd2string($cmd);
5006 }
5007
5008 sub vm_reset {
5009     my ($vmid, $skiplock) = @_;
5010
5011     PVE::QemuConfig->lock_config($vmid, sub {
5012
5013         my $conf = PVE::QemuConfig->load_config($vmid);
5014
5015         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5016
5017         vm_mon_cmd($vmid, "system_reset");
5018     });
5019 }
5020
5021 sub get_vm_volumes {
5022     my ($conf) = @_;
5023
5024     my $vollist = [];
5025     foreach_volid($conf, sub {
5026         my ($volid, $attr) = @_;
5027
5028         return if $volid =~ m|^/|;
5029
5030         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5031         return if !$sid;
5032
5033         push @$vollist, $volid;
5034     });
5035
5036     return $vollist;
5037 }
5038
5039 sub vm_stop_cleanup {
5040     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5041
5042     eval {
5043
5044         if (!$keepActive) {
5045             my $vollist = get_vm_volumes($conf);
5046             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5047         }
5048
5049         foreach my $ext (qw(mon qmp pid vnc qga)) {
5050             unlink "/var/run/qemu-server/${vmid}.$ext";
5051         }
5052
5053         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5054     };
5055     warn $@ if $@; # avoid errors - just warn
5056 }
5057
5058 # Note: use $nockeck to skip tests if VM configuration file exists.
5059 # We need that when migration VMs to other nodes (files already moved)
5060 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5061 sub vm_stop {
5062     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5063
5064     $force = 1 if !defined($force) && !$shutdown;
5065
5066     if ($migratedfrom){
5067         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5068         kill 15, $pid if $pid;
5069         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5070         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5071         return;
5072     }
5073
5074     PVE::QemuConfig->lock_config($vmid, sub {
5075
5076         my $pid = check_running($vmid, $nocheck);
5077         return if !$pid;
5078
5079         my $conf;
5080         if (!$nocheck) {
5081             $conf = PVE::QemuConfig->load_config($vmid);
5082             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5083             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5084                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5085                 $timeout = $opts->{down} if $opts->{down};
5086             }
5087         }
5088
5089         $timeout = 60 if !defined($timeout);
5090
5091         eval {
5092             if ($shutdown) {
5093                 if (defined($conf) && $conf->{agent}) {
5094                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5095                 } else {
5096                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5097                 }
5098             } else {
5099                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5100             }
5101         };
5102         my $err = $@;
5103
5104         if (!$err) {
5105             my $count = 0;
5106             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5107                 $count++;
5108                 sleep 1;
5109             }
5110
5111             if ($count >= $timeout) {
5112                 if ($force) {
5113                     warn "VM still running - terminating now with SIGTERM\n";
5114                     kill 15, $pid;
5115                 } else {
5116                     die "VM quit/powerdown failed - got timeout\n";
5117                 }
5118             } else {
5119                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5120                 return;
5121             }
5122         } else {
5123             if ($force) {
5124                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5125                 kill 15, $pid;
5126             } else {
5127                 die "VM quit/powerdown failed\n";
5128             }
5129         }
5130
5131         # wait again
5132         $timeout = 10;
5133
5134         my $count = 0;
5135         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5136             $count++;
5137             sleep 1;
5138         }
5139
5140         if ($count >= $timeout) {
5141             warn "VM still running - terminating now with SIGKILL\n";
5142             kill 9, $pid;
5143             sleep 1;
5144         }
5145
5146         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5147    });
5148 }
5149
5150 sub vm_suspend {
5151     my ($vmid, $skiplock) = @_;
5152
5153     PVE::QemuConfig->lock_config($vmid, sub {
5154
5155         my $conf = PVE::QemuConfig->load_config($vmid);
5156
5157         PVE::QemuConfig->check_lock($conf)
5158             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5159
5160         vm_mon_cmd($vmid, "stop");
5161     });
5162 }
5163
5164 sub vm_resume {
5165     my ($vmid, $skiplock, $nocheck) = @_;
5166
5167     PVE::QemuConfig->lock_config($vmid, sub {
5168
5169         if (!$nocheck) {
5170
5171             my $conf = PVE::QemuConfig->load_config($vmid);
5172
5173             PVE::QemuConfig->check_lock($conf)
5174                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5175
5176             vm_mon_cmd($vmid, "cont");
5177
5178         } else {
5179             vm_mon_cmd_nocheck($vmid, "cont");
5180         }
5181     });
5182 }
5183
5184 sub vm_sendkey {
5185     my ($vmid, $skiplock, $key) = @_;
5186
5187     PVE::QemuConfig->lock_config($vmid, sub {
5188
5189         my $conf = PVE::QemuConfig->load_config($vmid);
5190
5191         # there is no qmp command, so we use the human monitor command
5192         vm_human_monitor_command($vmid, "sendkey $key");
5193     });
5194 }
5195
5196 sub vm_destroy {
5197     my ($storecfg, $vmid, $skiplock) = @_;
5198
5199     PVE::QemuConfig->lock_config($vmid, sub {
5200
5201         my $conf = PVE::QemuConfig->load_config($vmid);
5202
5203         if (!check_running($vmid)) {
5204             destroy_vm($storecfg, $vmid, undef, $skiplock);
5205         } else {
5206             die "VM $vmid is running - destroy failed\n";
5207         }
5208     });
5209 }
5210
5211 # pci helpers
5212
5213 sub file_write {
5214     my ($filename, $buf) = @_;
5215
5216     my $fh = IO::File->new($filename, "w");
5217     return undef if !$fh;
5218
5219     my $res = print $fh $buf;
5220
5221     $fh->close();
5222
5223     return $res;
5224 }
5225
5226 sub pci_device_info {
5227     my ($name) = @_;
5228
5229     my $res;
5230
5231     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5232     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5233
5234     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5235     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5236
5237     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5238     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5239
5240     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5241     return undef if !defined($product) || $product !~ s/^0x//;
5242
5243     $res = {
5244         name => $name,
5245         vendor => $vendor,
5246         product => $product,
5247         domain => $domain,
5248         bus => $bus,
5249         slot => $slot,
5250         func => $func,
5251         irq => $irq,
5252         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5253     };
5254
5255     return $res;
5256 }
5257
5258 sub pci_dev_reset {
5259     my ($dev) = @_;
5260
5261     my $name = $dev->{name};
5262
5263     my $fn = "$pcisysfs/devices/$name/reset";
5264
5265     return file_write($fn, "1");
5266 }
5267
5268 sub pci_dev_bind_to_vfio {
5269     my ($dev) = @_;
5270
5271     my $name = $dev->{name};
5272
5273     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5274
5275     if (!-d $vfio_basedir) {
5276         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5277     }
5278     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5279
5280     my $testdir = "$vfio_basedir/$name";
5281     return 1 if -d $testdir;
5282
5283     my $data = "$dev->{vendor} $dev->{product}";
5284     return undef if !file_write("$vfio_basedir/new_id", $data);
5285
5286     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5287     if (!file_write($fn, $name)) {
5288         return undef if -f $fn;
5289     }
5290
5291     $fn = "$vfio_basedir/bind";
5292     if (! -d $testdir) {
5293         return undef if !file_write($fn, $name);
5294     }
5295
5296     return -d $testdir;
5297 }
5298
5299 sub pci_dev_group_bind_to_vfio {
5300     my ($pciid) = @_;
5301
5302     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5303
5304     if (!-d $vfio_basedir) {
5305         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5306     }
5307     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5308
5309     # get IOMMU group devices
5310     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5311       my @devs = grep /^0000:/, readdir($D);
5312     closedir($D);
5313
5314     foreach my $pciid (@devs) {
5315         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5316
5317         # pci bridges, switches or root ports are not supported
5318         # they have a pci_bus subdirectory so skip them
5319         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5320
5321         my $info = pci_device_info($1);
5322         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5323     }
5324
5325     return 1;
5326 }
5327
5328 # vzdump restore implementaion
5329
5330 sub tar_archive_read_firstfile {
5331     my $archive = shift;
5332
5333     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5334
5335     # try to detect archive type first
5336     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5337         die "unable to open file '$archive'\n";
5338     my $firstfile = <$fh>;
5339     kill 15, $pid;
5340     close $fh;
5341
5342     die "ERROR: archive contaions no data\n" if !$firstfile;
5343     chomp $firstfile;
5344
5345     return $firstfile;
5346 }
5347
5348 sub tar_restore_cleanup {
5349     my ($storecfg, $statfile) = @_;
5350
5351     print STDERR "starting cleanup\n";
5352
5353     if (my $fd = IO::File->new($statfile, "r")) {
5354         while (defined(my $line = <$fd>)) {
5355             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5356                 my $volid = $2;
5357                 eval {
5358                     if ($volid =~ m|^/|) {
5359                         unlink $volid || die 'unlink failed\n';
5360                     } else {
5361                         PVE::Storage::vdisk_free($storecfg, $volid);
5362                     }
5363                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5364                 };
5365                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5366             } else {
5367                 print STDERR "unable to parse line in statfile - $line";
5368             }
5369         }
5370         $fd->close();
5371     }
5372 }
5373
5374 sub restore_archive {
5375     my ($archive, $vmid, $user, $opts) = @_;
5376
5377     my $format = $opts->{format};
5378     my $comp;
5379
5380     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5381         $format = 'tar' if !$format;
5382         $comp = 'gzip';
5383     } elsif ($archive =~ m/\.tar$/) {
5384         $format = 'tar' if !$format;
5385     } elsif ($archive =~ m/.tar.lzo$/) {
5386         $format = 'tar' if !$format;
5387         $comp = 'lzop';
5388     } elsif ($archive =~ m/\.vma$/) {
5389         $format = 'vma' if !$format;
5390     } elsif ($archive =~ m/\.vma\.gz$/) {
5391         $format = 'vma' if !$format;
5392         $comp = 'gzip';
5393     } elsif ($archive =~ m/\.vma\.lzo$/) {
5394         $format = 'vma' if !$format;
5395         $comp = 'lzop';
5396     } else {
5397         $format = 'vma' if !$format; # default
5398     }
5399
5400     # try to detect archive format
5401     if ($format eq 'tar') {
5402         return restore_tar_archive($archive, $vmid, $user, $opts);
5403     } else {
5404         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5405     }
5406 }
5407
5408 sub restore_update_config_line {
5409     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5410
5411     return if $line =~ m/^\#qmdump\#/;
5412     return if $line =~ m/^\#vzdump\#/;
5413     return if $line =~ m/^lock:/;
5414     return if $line =~ m/^unused\d+:/;
5415     return if $line =~ m/^parent:/;
5416     return if $line =~ m/^template:/; # restored VM is never a template
5417
5418     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5419     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5420         # try to convert old 1.X settings
5421         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5422         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5423             my ($model, $macaddr) = split(/\=/, $devconfig);
5424             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5425             my $net = {
5426                 model => $model,
5427                 bridge => "vmbr$ind",
5428                 macaddr => $macaddr,
5429             };
5430             my $netstr = print_net($net);
5431
5432             print $outfd "net$cookie->{netcount}: $netstr\n";
5433             $cookie->{netcount}++;
5434         }
5435     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5436         my ($id, $netstr) = ($1, $2);
5437         my $net = parse_net($netstr);
5438         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5439         $netstr = print_net($net);
5440         print $outfd "$id: $netstr\n";
5441     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5442         my $virtdev = $1;
5443         my $value = $3;
5444         my $di = parse_drive($virtdev, $value);
5445         if (defined($di->{backup}) && !$di->{backup}) {
5446             print $outfd "#$line";
5447         } elsif ($map->{$virtdev}) {
5448             delete $di->{format}; # format can change on restore
5449             $di->{file} = $map->{$virtdev};
5450             $value = print_drive($vmid, $di);
5451             print $outfd "$virtdev: $value\n";
5452         } else {
5453             print $outfd $line;
5454         }
5455     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5456         my ($uuid, $uuid_str);
5457         UUID::generate($uuid);
5458         UUID::unparse($uuid, $uuid_str);
5459         my $smbios1 = parse_smbios1($2);
5460         $smbios1->{uuid} = $uuid_str;
5461         print $outfd $1.print_smbios1($smbios1)."\n";
5462     } else {
5463         print $outfd $line;
5464     }
5465 }
5466
5467 sub scan_volids {
5468     my ($cfg, $vmid) = @_;
5469
5470     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5471
5472     my $volid_hash = {};
5473     foreach my $storeid (keys %$info) {
5474         foreach my $item (@{$info->{$storeid}}) {
5475             next if !($item->{volid} && $item->{size});
5476             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5477             $volid_hash->{$item->{volid}} = $item;
5478         }
5479     }
5480
5481     return $volid_hash;
5482 }
5483
5484 sub is_volume_in_use {
5485     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5486
5487     my $path = PVE::Storage::path($storecfg, $volid);
5488
5489     my $scan_config = sub {
5490         my ($cref, $snapname) = @_;
5491
5492         foreach my $key (keys %$cref) {
5493             my $value = $cref->{$key};
5494             if (is_valid_drivename($key)) {
5495                 next if $skip_drive && $key eq $skip_drive;
5496                 my $drive = parse_drive($key, $value);
5497                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5498                 return 1 if $volid eq $drive->{file};
5499                 if ($drive->{file} =~ m!^/!) {
5500                     return 1 if $drive->{file} eq $path;
5501                 } else {
5502                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5503                     next if !$storeid;
5504                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5505                     next if !$scfg;
5506                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5507                 }
5508             }
5509         }
5510
5511         return 0;
5512     };
5513
5514     return 1 if &$scan_config($conf);
5515
5516     undef $skip_drive;
5517
5518     foreach my $snapname (keys %{$conf->{snapshots}}) {
5519         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5520     }
5521
5522     return 0;
5523 }
5524
5525 sub update_disksize {
5526     my ($vmid, $conf, $volid_hash) = @_;
5527
5528     my $changes;
5529
5530     # used and unused disks
5531     my $referenced = {};
5532
5533     # Note: it is allowed to define multiple storages with same path (alias), so
5534     # we need to check both 'volid' and real 'path' (two different volid can point
5535     # to the same path).
5536
5537     my $referencedpath = {};
5538
5539     # update size info
5540     foreach my $opt (keys %$conf) {
5541         if (is_valid_drivename($opt)) {
5542             my $drive = parse_drive($opt, $conf->{$opt});
5543             my $volid = $drive->{file};
5544             next if !$volid;
5545
5546             $referenced->{$volid} = 1;
5547             if ($volid_hash->{$volid} &&
5548                 (my $path = $volid_hash->{$volid}->{path})) {
5549                 $referencedpath->{$path} = 1;
5550             }
5551
5552             next if drive_is_cdrom($drive);
5553             next if !$volid_hash->{$volid};
5554
5555             $drive->{size} = $volid_hash->{$volid}->{size};
5556             my $new = print_drive($vmid, $drive);
5557             if ($new ne $conf->{$opt}) {
5558                 $changes = 1;
5559                 $conf->{$opt} = $new;
5560             }
5561         }
5562     }
5563
5564     # remove 'unusedX' entry if volume is used
5565     foreach my $opt (keys %$conf) {
5566         next if $opt !~ m/^unused\d+$/;
5567         my $volid = $conf->{$opt};
5568         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5569         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5570             $changes = 1;
5571             delete $conf->{$opt};
5572         }
5573
5574         $referenced->{$volid} = 1;
5575         $referencedpath->{$path} = 1 if $path;
5576     }
5577
5578     foreach my $volid (sort keys %$volid_hash) {
5579         next if $volid =~ m/vm-$vmid-state-/;
5580         next if $referenced->{$volid};
5581         my $path = $volid_hash->{$volid}->{path};
5582         next if !$path; # just to be sure
5583         next if $referencedpath->{$path};
5584         $changes = 1;
5585         PVE::QemuConfig->add_unused_volume($conf, $volid);
5586         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5587     }
5588
5589     return $changes;
5590 }
5591
5592 sub rescan {
5593     my ($vmid, $nolock) = @_;
5594
5595     my $cfg = PVE::Storage::config();
5596
5597     my $volid_hash = scan_volids($cfg, $vmid);
5598
5599     my $updatefn =  sub {
5600         my ($vmid) = @_;
5601
5602         my $conf = PVE::QemuConfig->load_config($vmid);
5603
5604         PVE::QemuConfig->check_lock($conf);
5605
5606         my $vm_volids = {};
5607         foreach my $volid (keys %$volid_hash) {
5608             my $info = $volid_hash->{$volid};
5609             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5610         }
5611
5612         my $changes = update_disksize($vmid, $conf, $vm_volids);
5613
5614         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5615     };
5616
5617     if (defined($vmid)) {
5618         if ($nolock) {
5619             &$updatefn($vmid);
5620         } else {
5621             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5622         }
5623     } else {
5624         my $vmlist = config_list();
5625         foreach my $vmid (keys %$vmlist) {
5626             if ($nolock) {
5627                 &$updatefn($vmid);
5628             } else {
5629                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5630             }
5631         }
5632     }
5633 }
5634
5635 sub restore_vma_archive {
5636     my ($archive, $vmid, $user, $opts, $comp) = @_;
5637
5638     my $input = $archive eq '-' ? "<&STDIN" : undef;
5639     my $readfrom = $archive;
5640
5641     my $uncomp = '';
5642     if ($comp) {
5643         $readfrom = '-';
5644         my $qarchive = PVE::Tools::shellquote($archive);
5645         if ($comp eq 'gzip') {
5646             $uncomp = "zcat $qarchive|";
5647         } elsif ($comp eq 'lzop') {
5648             $uncomp = "lzop -d -c $qarchive|";
5649         } else {
5650             die "unknown compression method '$comp'\n";
5651         }
5652
5653     }
5654
5655     my $tmpdir = "/var/tmp/vzdumptmp$$";
5656     rmtree $tmpdir;
5657
5658     # disable interrupts (always do cleanups)
5659     local $SIG{INT} =
5660         local $SIG{TERM} =
5661         local $SIG{QUIT} =
5662         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5663
5664     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5665     POSIX::mkfifo($mapfifo, 0600);
5666     my $fifofh;
5667
5668     my $openfifo = sub {
5669         open($fifofh, '>', $mapfifo) || die $!;
5670     };
5671
5672     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5673
5674     my $oldtimeout;
5675     my $timeout = 5;
5676
5677     my $devinfo = {};
5678
5679     my $rpcenv = PVE::RPCEnvironment::get();
5680
5681     my $conffile = PVE::QemuConfig->config_file($vmid);
5682     my $tmpfn = "$conffile.$$.tmp";
5683
5684     # Note: $oldconf is undef if VM does not exists
5685     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5686     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5687
5688     my $print_devmap = sub {
5689         my $virtdev_hash = {};
5690
5691         my $cfgfn = "$tmpdir/qemu-server.conf";
5692
5693         # we can read the config - that is already extracted
5694         my $fh = IO::File->new($cfgfn, "r") ||
5695             "unable to read qemu-server.conf - $!\n";
5696
5697         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5698         if (-f $fwcfgfn) {
5699             my $pve_firewall_dir = '/etc/pve/firewall';
5700             mkdir $pve_firewall_dir; # make sure the dir exists
5701             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5702         }
5703
5704         while (defined(my $line = <$fh>)) {
5705             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5706                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5707                 die "archive does not contain data for drive '$virtdev'\n"
5708                     if !$devinfo->{$devname};
5709                 if (defined($opts->{storage})) {
5710                     $storeid = $opts->{storage} || 'local';
5711                 } elsif (!$storeid) {
5712                     $storeid = 'local';
5713                 }
5714                 $format = 'raw' if !$format;
5715                 $devinfo->{$devname}->{devname} = $devname;
5716                 $devinfo->{$devname}->{virtdev} = $virtdev;
5717                 $devinfo->{$devname}->{format} = $format;
5718                 $devinfo->{$devname}->{storeid} = $storeid;
5719
5720                 # check permission on storage
5721                 my $pool = $opts->{pool}; # todo: do we need that?
5722                 if ($user ne 'root@pam') {
5723                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5724                 }
5725
5726                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5727             }
5728         }
5729
5730         foreach my $devname (keys %$devinfo) {
5731             die "found no device mapping information for device '$devname'\n"
5732                 if !$devinfo->{$devname}->{virtdev};
5733         }
5734
5735         my $cfg = PVE::Storage::config();
5736
5737         # create empty/temp config
5738         if ($oldconf) {
5739             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5740             foreach_drive($oldconf, sub {
5741                 my ($ds, $drive) = @_;
5742
5743                 return if drive_is_cdrom($drive);
5744
5745                 my $volid = $drive->{file};
5746
5747                 return if !$volid || $volid =~ m|^/|;
5748
5749                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5750                 return if !$path || !$owner || ($owner != $vmid);
5751
5752                 # Note: only delete disk we want to restore
5753                 # other volumes will become unused
5754                 if ($virtdev_hash->{$ds}) {
5755                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5756                     if (my $err = $@) {
5757                         warn $err;
5758                     }
5759                 }
5760             });
5761
5762             # delete vmstate files
5763             # since after the restore we have no snapshots anymore
5764             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5765                 my $snap = $oldconf->{snapshots}->{$snapname};
5766                 if ($snap->{vmstate}) {
5767                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5768                     if (my $err = $@) {
5769                         warn $err;
5770                     }
5771                 }
5772             }
5773         }
5774
5775         my $map = {};
5776         foreach my $virtdev (sort keys %$virtdev_hash) {
5777             my $d = $virtdev_hash->{$virtdev};
5778             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5779             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5780
5781             # test if requested format is supported
5782             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5783             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5784             $d->{format} = $defFormat if !$supported;
5785
5786             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5787                                                   $d->{format}, undef, $alloc_size);
5788             print STDERR "new volume ID is '$volid'\n";
5789             $d->{volid} = $volid;
5790             my $path = PVE::Storage::path($cfg, $volid);
5791
5792             PVE::Storage::activate_volumes($cfg,[$volid]);
5793
5794             my $write_zeros = 1;
5795             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5796                 $write_zeros = 0;
5797             }
5798
5799             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5800
5801             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5802             $map->{$virtdev} = $volid;
5803         }
5804
5805         $fh->seek(0, 0) || die "seek failed - $!\n";
5806
5807         my $outfd = new IO::File ($tmpfn, "w") ||
5808             die "unable to write config for VM $vmid\n";
5809
5810         my $cookie = { netcount => 0 };
5811         while (defined(my $line = <$fh>)) {
5812             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5813         }
5814
5815         $fh->close();
5816         $outfd->close();
5817     };
5818
5819     eval {
5820         # enable interrupts
5821         local $SIG{INT} =
5822             local $SIG{TERM} =
5823             local $SIG{QUIT} =
5824             local $SIG{HUP} =
5825             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5826         local $SIG{ALRM} = sub { die "got timeout\n"; };
5827
5828         $oldtimeout = alarm($timeout);
5829
5830         my $parser = sub {
5831             my $line = shift;
5832
5833             print "$line\n";
5834
5835             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5836                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5837                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5838             } elsif ($line =~ m/^CTIME: /) {
5839                 # we correctly received the vma config, so we can disable
5840                 # the timeout now for disk allocation (set to 10 minutes, so
5841                 # that we always timeout if something goes wrong)
5842                 alarm(600);
5843                 &$print_devmap();
5844                 print $fifofh "done\n";
5845                 my $tmp = $oldtimeout || 0;
5846                 $oldtimeout = undef;
5847                 alarm($tmp);
5848                 close($fifofh);
5849             }
5850         };
5851
5852         print "restore vma archive: $cmd\n";
5853         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5854     };
5855     my $err = $@;
5856
5857     alarm($oldtimeout) if $oldtimeout;
5858
5859     my $vollist = [];
5860     foreach my $devname (keys %$devinfo) {
5861         my $volid = $devinfo->{$devname}->{volid};
5862         push @$vollist, $volid if $volid;
5863     }
5864
5865     my $cfg = PVE::Storage::config();
5866     PVE::Storage::deactivate_volumes($cfg, $vollist);
5867
5868     unlink $mapfifo;
5869
5870     if ($err) {
5871         rmtree $tmpdir;
5872         unlink $tmpfn;
5873
5874         foreach my $devname (keys %$devinfo) {
5875             my $volid = $devinfo->{$devname}->{volid};
5876             next if !$volid;
5877             eval {
5878                 if ($volid =~ m|^/|) {
5879                     unlink $volid || die 'unlink failed\n';
5880                 } else {
5881                     PVE::Storage::vdisk_free($cfg, $volid);
5882                 }
5883                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5884             };
5885             print STDERR "unable to cleanup '$volid' - $@" if $@;
5886         }
5887         die $err;
5888     }
5889
5890     rmtree $tmpdir;
5891
5892     rename($tmpfn, $conffile) ||
5893         die "unable to commit configuration file '$conffile'\n";
5894
5895     PVE::Cluster::cfs_update(); # make sure we read new file
5896
5897     eval { rescan($vmid, 1); };
5898     warn $@ if $@;
5899 }
5900
5901 sub restore_tar_archive {
5902     my ($archive, $vmid, $user, $opts) = @_;
5903
5904     if ($archive ne '-') {
5905         my $firstfile = tar_archive_read_firstfile($archive);
5906         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5907             if $firstfile ne 'qemu-server.conf';
5908     }
5909
5910     my $storecfg = PVE::Storage::config();
5911
5912     # destroy existing data - keep empty config
5913     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5914     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5915
5916     my $tocmd = "/usr/lib/qemu-server/qmextract";
5917
5918     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5919     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5920     $tocmd .= ' --prealloc' if $opts->{prealloc};
5921     $tocmd .= ' --info' if $opts->{info};
5922
5923     # tar option "xf" does not autodetect compression when read from STDIN,
5924     # so we pipe to zcat
5925     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5926         PVE::Tools::shellquote("--to-command=$tocmd");
5927
5928     my $tmpdir = "/var/tmp/vzdumptmp$$";
5929     mkpath $tmpdir;
5930
5931     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5932     local $ENV{VZDUMP_VMID} = $vmid;
5933     local $ENV{VZDUMP_USER} = $user;
5934
5935     my $conffile = PVE::QemuConfig->config_file($vmid);
5936     my $tmpfn = "$conffile.$$.tmp";
5937
5938     # disable interrupts (always do cleanups)
5939     local $SIG{INT} =
5940         local $SIG{TERM} =
5941         local $SIG{QUIT} =
5942         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5943
5944     eval {
5945         # enable interrupts
5946         local $SIG{INT} =
5947             local $SIG{TERM} =
5948             local $SIG{QUIT} =
5949             local $SIG{HUP} =
5950             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5951
5952         if ($archive eq '-') {
5953             print "extracting archive from STDIN\n";
5954             run_command($cmd, input => "<&STDIN");
5955         } else {
5956             print "extracting archive '$archive'\n";
5957             run_command($cmd);
5958         }
5959
5960         return if $opts->{info};
5961
5962         # read new mapping
5963         my $map = {};
5964         my $statfile = "$tmpdir/qmrestore.stat";
5965         if (my $fd = IO::File->new($statfile, "r")) {
5966             while (defined (my $line = <$fd>)) {
5967                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5968                     $map->{$1} = $2 if $1;
5969                 } else {
5970                     print STDERR "unable to parse line in statfile - $line\n";
5971                 }
5972             }
5973             $fd->close();
5974         }
5975
5976         my $confsrc = "$tmpdir/qemu-server.conf";
5977
5978         my $srcfd = new IO::File($confsrc, "r") ||
5979             die "unable to open file '$confsrc'\n";
5980
5981         my $outfd = new IO::File ($tmpfn, "w") ||
5982             die "unable to write config for VM $vmid\n";
5983
5984         my $cookie = { netcount => 0 };
5985         while (defined (my $line = <$srcfd>)) {
5986             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5987         }
5988
5989         $srcfd->close();
5990         $outfd->close();
5991     };
5992     my $err = $@;
5993
5994     if ($err) {
5995
5996         unlink $tmpfn;
5997
5998         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
5999
6000         die $err;
6001     }
6002
6003     rmtree $tmpdir;
6004
6005     rename $tmpfn, $conffile ||
6006         die "unable to commit configuration file '$conffile'\n";
6007
6008     PVE::Cluster::cfs_update(); # make sure we read new file
6009
6010     eval { rescan($vmid, 1); };
6011     warn $@ if $@;
6012 };
6013
6014 sub foreach_storage_used_by_vm {
6015     my ($conf, $func) = @_;
6016
6017     my $sidhash = {};
6018
6019     foreach_drive($conf, sub {
6020         my ($ds, $drive) = @_;
6021         return if drive_is_cdrom($drive);
6022
6023         my $volid = $drive->{file};
6024
6025         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6026         $sidhash->{$sid} = $sid if $sid;
6027     });
6028
6029     foreach my $sid (sort keys %$sidhash) {
6030         &$func($sid);
6031     }
6032 }
6033
6034 sub do_snapshots_with_qemu {
6035     my ($storecfg, $volid) = @_;
6036
6037     my $storage_name = PVE::Storage::parse_volume_id($volid);
6038
6039     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6040         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6041         return 1;
6042     }
6043
6044     if ($volid =~ m/\.(qcow2|qed)$/){
6045         return 1;
6046     }
6047
6048     return undef;
6049 }
6050
6051 sub qga_check_running {
6052     my ($vmid) = @_;
6053
6054     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6055     if ($@) {
6056         warn "Qemu Guest Agent is not running - $@";
6057         return 0;
6058     }
6059     return 1;
6060 }
6061
6062 sub template_create {
6063     my ($vmid, $conf, $disk) = @_;
6064
6065     my $storecfg = PVE::Storage::config();
6066
6067     foreach_drive($conf, sub {
6068         my ($ds, $drive) = @_;
6069
6070         return if drive_is_cdrom($drive);
6071         return if $disk && $ds ne $disk;
6072
6073         my $volid = $drive->{file};
6074         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6075
6076         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6077         $drive->{file} = $voliddst;
6078         $conf->{$ds} = print_drive($vmid, $drive);
6079         PVE::QemuConfig->write_config($vmid, $conf);
6080     });
6081 }
6082
6083 sub qemu_img_convert {
6084     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6085
6086     my $storecfg = PVE::Storage::config();
6087     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6088     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6089
6090     if ($src_storeid && $dst_storeid) {
6091
6092         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6093
6094         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6095         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6096
6097         my $src_format = qemu_img_format($src_scfg, $src_volname);
6098         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6099
6100         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6101         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6102
6103         my $cmd = [];
6104         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6105         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6106         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6107         if ($is_zero_initialized) {
6108             push @$cmd, "zeroinit:$dst_path";
6109         } else {
6110             push @$cmd, $dst_path;
6111         }
6112
6113         my $parser = sub {
6114             my $line = shift;
6115             if($line =~ m/\((\S+)\/100\%\)/){
6116                 my $percent = $1;
6117                 my $transferred = int($size * $percent / 100);
6118                 my $remaining = $size - $transferred;
6119
6120                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6121             }
6122
6123         };
6124
6125         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6126         my $err = $@;
6127         die "copy failed: $err" if $err;
6128     }
6129 }
6130
6131 sub qemu_img_format {
6132     my ($scfg, $volname) = @_;
6133
6134     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6135         return $1;
6136     } else {
6137         return "raw";
6138     }
6139 }
6140
6141 sub qemu_drive_mirror {
6142     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6143
6144     $jobs = {} if !$jobs;
6145
6146     my $qemu_target;
6147     my $format;
6148     $jobs->{"drive-$drive"} = {};
6149
6150     if ($dst_volid =~ /^nbd:/) {
6151         $qemu_target = $dst_volid;
6152         $format = "nbd";
6153     } else {
6154         my $storecfg = PVE::Storage::config();
6155         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6156
6157         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6158
6159         $format = qemu_img_format($dst_scfg, $dst_volname);
6160
6161         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6162
6163         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6164     }
6165
6166     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6167     $opts->{format} = $format if $format;
6168
6169     print "drive mirror is starting for drive-$drive\n";
6170
6171     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6172
6173     if (my $err = $@) {
6174         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6175         die "mirroring error: $err";
6176     }
6177
6178     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6179 }
6180
6181 sub qemu_drive_mirror_monitor {
6182     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6183
6184     eval {
6185         my $err_complete = 0;
6186
6187         while (1) {
6188             die "storage migration timed out\n" if $err_complete > 300;
6189
6190             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6191
6192             my $running_mirror_jobs = {};
6193             foreach my $stat (@$stats) {
6194                 next if $stat->{type} ne 'mirror';
6195                 $running_mirror_jobs->{$stat->{device}} = $stat;
6196             }
6197
6198             my $readycounter = 0;
6199
6200             foreach my $job (keys %$jobs) {
6201
6202                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6203                     print "$job : finished\n";
6204                     delete $jobs->{$job};
6205                     next;
6206                 }
6207
6208                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6209
6210                 my $busy = $running_mirror_jobs->{$job}->{busy};
6211                 my $ready = $running_mirror_jobs->{$job}->{ready};
6212                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6213                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6214                     my $remaining = $total - $transferred;
6215                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6216
6217                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6218                 }
6219
6220                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6221             }
6222
6223             last if scalar(keys %$jobs) == 0;
6224
6225             if ($readycounter == scalar(keys %$jobs)) {
6226                 print "all mirroring jobs are ready \n";
6227                 last if $skipcomplete; #do the complete later
6228
6229                 if ($vmiddst && $vmiddst != $vmid) {
6230                     my $agent_running = $qga && qga_check_running($vmid);
6231                     if ($agent_running) {
6232                         print "freeze filesystem\n";
6233                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6234                     } else {
6235                         print "suspend vm\n";
6236                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6237                     }
6238
6239                     # if we clone a disk for a new target vm, we don't switch the disk
6240                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6241
6242                     if ($agent_running) {
6243                         print "unfreeze filesystem\n";
6244                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6245                     } else {
6246                         print "resume vm\n";
6247                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6248                     }
6249
6250                     last;
6251                 } else {
6252
6253                     foreach my $job (keys %$jobs) {
6254                         # try to switch the disk if source and destination are on the same guest
6255                         print "$job: Completing block job...\n";
6256
6257                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6258                         if ($@ =~ m/cannot be completed/) {
6259                             print "$job: Block job cannot be completed, try again.\n";
6260                             $err_complete++;
6261                         }else {
6262                             print "$job: Completed successfully.\n";
6263                             $jobs->{$job}->{complete} = 1;
6264                         }
6265                     }
6266                 }
6267             }
6268             sleep 1;
6269         }
6270     };
6271     my $err = $@;
6272
6273     if ($err) {
6274         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6275         die "mirroring error: $err";
6276     }
6277
6278 }
6279
6280 sub qemu_blockjobs_cancel {
6281     my ($vmid, $jobs) = @_;
6282
6283     foreach my $job (keys %$jobs) {
6284         print "$job: Cancelling block job\n";
6285         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6286         $jobs->{$job}->{cancel} = 1;
6287     }
6288
6289     while (1) {
6290         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6291
6292         my $running_jobs = {};
6293         foreach my $stat (@$stats) {
6294             $running_jobs->{$stat->{device}} = $stat;
6295         }
6296
6297         foreach my $job (keys %$jobs) {
6298
6299             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6300                 print "$job: Done.\n";
6301                 delete $jobs->{$job};
6302             }
6303         }
6304
6305         last if scalar(keys %$jobs) == 0;
6306
6307         sleep 1;
6308     }
6309 }
6310
6311 sub clone_disk {
6312     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6313         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6314
6315     my $newvolid;
6316
6317     if (!$full) {
6318         print "create linked clone of drive $drivename ($drive->{file})\n";
6319         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6320         push @$newvollist, $newvolid;
6321     } else {
6322
6323         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6324         $storeid = $storage if $storage;
6325
6326         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6327         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6328
6329         print "create full clone of drive $drivename ($drive->{file})\n";
6330         my $name = undef;
6331         if (drive_is_cloudinit($drive)) {
6332             $name = "vm-$newvmid-cloudinit";
6333             # cloudinit only supports raw and qcow2 atm:
6334             if ($dst_format eq 'qcow2') {
6335                 $name .= '.qcow2';
6336             } elsif ($dst_format ne 'raw') {
6337                 die "clone: unhandled format for cloudinit image\n";
6338             }
6339         }
6340         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6341         push @$newvollist, $newvolid;
6342
6343         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6344
6345         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6346         if (!$running || $snapname) {
6347             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6348         } else {
6349
6350             my $kvmver = get_running_qemu_version ($vmid);
6351             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6352                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6353                     if $drive->{iothread};
6354             }
6355
6356             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6357         }
6358     }
6359
6360     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6361
6362     my $disk = $drive;
6363     $disk->{format} = undef;
6364     $disk->{file} = $newvolid;
6365     $disk->{size} = $size;
6366
6367     return $disk;
6368 }
6369
6370 # this only works if VM is running
6371 sub get_current_qemu_machine {
6372     my ($vmid) = @_;
6373
6374     my $cmd = { execute => 'query-machines', arguments => {} };
6375     my $res = vm_qmp_command($vmid, $cmd);
6376
6377     my ($current, $default);
6378     foreach my $e (@$res) {
6379         $default = $e->{name} if $e->{'is-default'};
6380         $current = $e->{name} if $e->{'is-current'};
6381     }
6382
6383     # fallback to the default machine if current is not supported by qemu
6384     return $current || $default || 'pc';
6385 }
6386
6387 sub get_running_qemu_version {
6388     my ($vmid) = @_;
6389     my $cmd = { execute => 'query-version', arguments => {} };
6390     my $res = vm_qmp_command($vmid, $cmd);
6391     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6392 }
6393
6394 sub qemu_machine_feature_enabled {
6395     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6396
6397     my $current_major;
6398     my $current_minor;
6399
6400     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6401
6402         $current_major = $3;
6403         $current_minor = $4;
6404
6405     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6406
6407         $current_major = $1;
6408         $current_minor = $2;
6409     }
6410
6411     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6412
6413
6414 }
6415
6416 sub qemu_machine_pxe {
6417     my ($vmid, $conf, $machine) = @_;
6418
6419     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6420
6421     foreach my $opt (keys %$conf) {
6422         next if $opt !~ m/^net(\d+)$/;
6423         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6424         next if !$net;
6425         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6426         return $machine.".pxe" if $romfile =~ m/pxe/;
6427         last;
6428     }
6429
6430     return $machine;
6431 }
6432
6433 sub qemu_use_old_bios_files {
6434     my ($machine_type) = @_;
6435
6436     return if !$machine_type;
6437
6438     my $use_old_bios_files = undef;
6439
6440     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6441         $machine_type = $1;
6442         $use_old_bios_files = 1;
6443     } else {
6444         my $kvmver = kvm_user_version();
6445         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6446         # load new efi bios files on migration. So this hack is required to allow
6447         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6448         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6449         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6450     }
6451
6452     return ($use_old_bios_files, $machine_type);
6453 }
6454
6455 sub create_efidisk {
6456     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6457
6458     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6459
6460     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6461     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6462     PVE::Storage::activate_volumes($storecfg, [$volid]);
6463
6464     my $path = PVE::Storage::path($storecfg, $volid);
6465     eval {
6466         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6467     };
6468     die "Copying EFI vars image failed: $@" if $@;
6469
6470     return ($volid, $vars_size);
6471 }
6472
6473 sub lspci {
6474
6475     my $devices = {};
6476
6477     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6478             my (undef, $id, $function) = @_;
6479             my $res = { id => $id, function => $function};
6480             push @{$devices->{$id}}, $res;
6481     });
6482
6483     # Entries should be sorted by functions.
6484     foreach my $id (keys %$devices) {
6485         my $dev = $devices->{$id};
6486         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6487     }
6488
6489     return $devices;
6490 }
6491
6492 sub vm_iothreads_list {
6493     my ($vmid) = @_;
6494
6495     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6496
6497     my $iothreads = {};
6498     foreach my $iothread (@$res) {
6499         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6500     }
6501
6502     return $iothreads;
6503 }
6504
6505 sub scsihw_infos {
6506     my ($conf, $drive) = @_;
6507
6508     my $maxdev = 0;
6509
6510     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6511         $maxdev = 7;
6512     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6513         $maxdev = 1;
6514     } else {
6515         $maxdev = 256;
6516     }
6517
6518     my $controller = int($drive->{index} / $maxdev);
6519     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6520
6521     return ($maxdev, $controller, $controller_prefix);
6522 }
6523
6524 sub add_hyperv_enlightenments {
6525     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6526
6527     return if $winversion < 6;
6528     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6529
6530     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6531
6532     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6533         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6534         push @$cpuFlags , 'hv_vapic';
6535         push @$cpuFlags , 'hv_time';
6536     } else {
6537         push @$cpuFlags , 'hv_spinlocks=0xffff';
6538     }
6539
6540     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6541         push @$cpuFlags , 'hv_reset';
6542         push @$cpuFlags , 'hv_vpindex';
6543         push @$cpuFlags , 'hv_runtime';
6544     }
6545
6546     if ($winversion >= 7) {
6547         push @$cpuFlags , 'hv_relaxed';
6548     }
6549 }
6550
6551 sub windows_version {
6552     my ($ostype) = @_;
6553
6554     return 0 if !$ostype;
6555
6556     my $winversion = 0;
6557
6558     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6559         $winversion = 5;
6560     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6561         $winversion = 6;
6562     } elsif ($ostype =~ m/^win(\d+)$/) {
6563         $winversion = $1;
6564     }
6565
6566     return $winversion;
6567 }
6568
6569 sub resolve_dst_disk_format {
6570         my ($storecfg, $storeid, $src_volname, $format) = @_;
6571         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6572
6573         if (!$format) {
6574             # if no target format is specified, use the source disk format as hint
6575             if ($src_volname) {
6576                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6577                 $format = qemu_img_format($scfg, $src_volname);
6578             } else {
6579                 return $defFormat;
6580             }
6581         }
6582
6583         # test if requested format is supported - else use default
6584         my $supported = grep { $_ eq $format } @$validFormats;
6585         $format = $defFormat if !$supported;
6586         return $format;
6587 }
6588
6589 sub resolve_first_disk {
6590     my $conf = shift;
6591     my @disks = PVE::QemuServer::valid_drive_names();
6592     my $firstdisk;
6593     foreach my $ds (reverse @disks) {
6594         next if !$conf->{$ds};
6595         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6596         next if PVE::QemuServer::drive_is_cdrom($disk);
6597         $firstdisk = $ds;
6598     }
6599     return $firstdisk;
6600 }
6601
6602 sub generate_smbios1_uuid {
6603     my ($uuid, $uuid_str);
6604     UUID::generate($uuid);
6605     UUID::unparse($uuid, $uuid_str);
6606     return "uuid=$uuid_str";
6607 }
6608
6609 # bash completion helper
6610
6611 sub complete_backup_archives {
6612     my ($cmdname, $pname, $cvalue) = @_;
6613
6614     my $cfg = PVE::Storage::config();
6615
6616     my $storeid;
6617
6618     if ($cvalue =~ m/^([^:]+):/) {
6619         $storeid = $1;
6620     }
6621
6622     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6623
6624     my $res = [];
6625     foreach my $id (keys %$data) {
6626         foreach my $item (@{$data->{$id}}) {
6627             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6628             push @$res, $item->{volid} if defined($item->{volid});
6629         }
6630     }
6631
6632     return $res;
6633 }
6634
6635 my $complete_vmid_full = sub {
6636     my ($running) = @_;
6637
6638     my $idlist = vmstatus();
6639
6640     my $res = [];
6641
6642     foreach my $id (keys %$idlist) {
6643         my $d = $idlist->{$id};
6644         if (defined($running)) {
6645             next if $d->{template};
6646             next if $running && $d->{status} ne 'running';
6647             next if !$running && $d->{status} eq 'running';
6648         }
6649         push @$res, $id;
6650
6651     }
6652     return $res;
6653 };
6654
6655 sub complete_vmid {
6656     return &$complete_vmid_full();
6657 }
6658
6659 sub complete_vmid_stopped {
6660     return &$complete_vmid_full(0);
6661 }
6662
6663 sub complete_vmid_running {
6664     return &$complete_vmid_full(1);
6665 }
6666
6667 sub complete_storage {
6668
6669     my $cfg = PVE::Storage::config();
6670     my $ids = $cfg->{ids};
6671
6672     my $res = [];
6673     foreach my $sid (keys %$ids) {
6674         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6675         next if !$ids->{$sid}->{content}->{images};
6676         push @$res, $sid;
6677     }
6678
6679     return $res;
6680 }
6681
6682 1;