PVE/QemuServer.pm

   1 package PVE::QemuServer;
   2
   3 use strict;
   4 use warnings;
   5 use POSIX;
   6 use IO::Handle;
   7 use IO::Select;
   8 use IO::File;
   9 use IO::Dir;
  10 use IO::Socket::UNIX;
  11 use File::Basename;
  12 use File::Path;
  13 use File::stat;
  14 use Getopt::Long;
  15 use Digest::SHA;
  16 use Fcntl ':flock';
  17 use Cwd 'abs_path';
  18 use IPC::Open3;
  19 use JSON;
  20 use Fcntl;
  21 use PVE::SafeSyslog;
  22 use Storable qw(dclone);
  23 use PVE::Exception qw(raise raise_param_exc);
  24 use PVE::Storage;
  25 use PVE::Tools qw(run_command lock_file lock_file_full file_read_firstline dir_glob_foreach $IPV6RE);
  26 use PVE::JSONSchema qw(get_standard_option);
  27 use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file);
  28 use PVE::INotify;
  29 use PVE::ProcFSTools;
  30 use PVE::QemuConfig;
  31 use PVE::QMPClient;
  32 use PVE::RPCEnvironment;
  33 use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr);
  34 use PVE::QemuServer::Memory;
  35 use PVE::QemuServer::USB qw(parse_usb_device);
  36 use PVE::QemuServer::Cloudinit;
  37 use Time::HiRes qw(gettimeofday);
  38 use File::Copy qw(copy);
  39 use URI::Escape;
  40
  41 my $OVMF_CODE = '/usr/share/kvm/OVMF_CODE-pure-efi.fd';
  42 my $OVMF_VARS = '/usr/share/kvm/OVMF_VARS-pure-efi.fd';
  43
  44 my $qemu_snap_storage = {rbd => 1, sheepdog => 1};
  45
  46 my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
  47
  48 my $QEMU_FORMAT_RE = qr/raw|cow|qcow|qcow2|qed|vmdk|cloop/;
  49
  50 # Note about locking: we use flock on the config file protect
  51 # against concurent actions.
  52 # Aditionaly, we have a 'lock' setting in the config file. This
  53 # can be set to 'migrate', 'backup', 'snapshot' or 'rollback'. Most actions are not
  54 # allowed when such lock is set. But you can ignore this kind of
  55 # lock with the --skiplock flag.
  56
  57 cfs_register_file('/qemu-server/',
  58                   \&parse_vm_config,
  59                   \&write_vm_config);
  60
  61 PVE::JSONSchema::register_standard_option('pve-qm-stateuri', {
  62     description => "Some command save/restore state from this location.",
  63     type => 'string',
  64     maxLength => 128,
  65     optional => 1,
  66 });
  67
  68 PVE::JSONSchema::register_standard_option('pve-snapshot-name', {
  69     description => "The name of the snapshot.",
  70     type => 'string', format => 'pve-configid',
  71     maxLength => 40,
  72 });
  73
  74 PVE::JSONSchema::register_standard_option('pve-qm-image-format', {
  75     type => 'string',
  76     enum => [qw(raw cow qcow qed qcow2 vmdk cloop)],
  77     description => "The drive's backing file's data format.",
  78     optional => 1,
  79 });
  80
  81 #no warnings 'redefine';
  82
  83 sub cgroups_write {
  84    my ($controller, $vmid, $option, $value) = @_;
  85
  86    my $path = "/sys/fs/cgroup/$controller/qemu.slice/$vmid.scope/$option";
  87    PVE::ProcFSTools::write_proc_entry($path, $value);
  88
  89 }
  90
  91 my $nodename = PVE::INotify::nodename();
  92
  93 mkdir "/etc/pve/nodes/$nodename";
  94 my $confdir = "/etc/pve/nodes/$nodename/qemu-server";
  95 mkdir $confdir;
  96
  97 my $var_run_tmpdir = "/var/run/qemu-server";
  98 mkdir $var_run_tmpdir;
  99
 100 my $lock_dir = "/var/lock/qemu-server";
 101 mkdir $lock_dir;
 102
 103 my $pcisysfs = "/sys/bus/pci";
 104
 105 my $cpu_vendor_list = {
 106     # Intel CPUs
 107     486 => 'GenuineIntel',
 108     pentium => 'GenuineIntel',
 109     pentium2  => 'GenuineIntel',
 110     pentium3  => 'GenuineIntel',
 111     coreduo => 'GenuineIntel',
 112     core2duo => 'GenuineIntel',
 113     Conroe  => 'GenuineIntel',
 114     Penryn  => 'GenuineIntel',
 115     Nehalem  => 'GenuineIntel',
 116     'Nehalem-IBRS'  => 'GenuineIntel',
 117     Westmere => 'GenuineIntel',
 118     'Westmere-IBRS' => 'GenuineIntel',
 119     SandyBridge => 'GenuineIntel',
 120     'SandyBridge-IBRS' => 'GenuineIntel',
 121     IvyBridge => 'GenuineIntel',
 122     'IvyBridge-IBRS' => 'GenuineIntel',
 123     Haswell => 'GenuineIntel',
 124     'Haswell-IBRS' => 'GenuineIntel',
 125     'Haswell-noTSX' => 'GenuineIntel',
 126     'Haswell-noTSX-IBRS' => 'GenuineIntel',
 127     Broadwell => 'GenuineIntel',
 128     'Broadwell-IBRS' => 'GenuineIntel',
 129     'Broadwell-noTSX' => 'GenuineIntel',
 130     'Broadwell-noTSX-IBRS' => 'GenuineIntel',
 131     'Skylake-Client' => 'GenuineIntel',
 132     'Skylake-Client-IBRS' => 'GenuineIntel',
 133     'Skylake-Server' => 'GenuineIntel',
 134     'Skylake-Server-IBRS' => 'GenuineIntel',
 135
 136     # AMD CPUs
 137     athlon => 'AuthenticAMD',
 138     phenom  => 'AuthenticAMD',
 139     Opteron_G1  => 'AuthenticAMD',
 140     Opteron_G2  => 'AuthenticAMD',
 141     Opteron_G3  => 'AuthenticAMD',
 142     Opteron_G4  => 'AuthenticAMD',
 143     Opteron_G5  => 'AuthenticAMD',
 144     EPYC => 'AuthenticAMD',
 145     'EPYC-IBPB' => 'AuthenticAMD',
 146
 147     # generic types, use vendor from host node
 148     host => 'default',
 149     kvm32 => 'default',
 150     kvm64 => 'default',
 151     qemu32 => 'default',
 152     qemu64 => 'default',
 153     max => 'default',
 154 };
 155
 156 my $cpu_flag = qr/[+-](pcid|spec-ctrl)/;
 157
 158 my $cpu_fmt = {
 159     cputype => {
 160         description => "Emulated CPU type.",
 161         type => 'string',
 162         enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
 163         default => 'kvm64',
 164         default_key => 1,
 165     },
 166     hidden => {
 167         description => "Do not identify as a KVM virtual machine.",
 168         type => 'boolean',
 169         optional => 1,
 170         default => 0
 171     },
 172     flags => {
 173         description => "List of additional CPU flags separated by ';'."
 174                      . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
 175                      . " Currently supported flags: 'pcid', 'spec-ctrl'.",
 176         format_description => '+FLAG[;-FLAG...]',
 177         type => 'string',
 178         pattern => qr/$cpu_flag(;$cpu_flag)*/,
 179         optional => 1,
 180     },
 181 };
 182
 183 my $watchdog_fmt = {
 184     model => {
 185         default_key => 1,
 186         type => 'string',
 187         enum => [qw(i6300esb ib700)],
 188         description => "Watchdog type to emulate.",
 189         default => 'i6300esb',
 190         optional => 1,
 191     },
 192     action => {
 193         type => 'string',
 194         enum => [qw(reset shutdown poweroff pause debug none)],
 195         description => "The action to perform if after activation the guest fails to poll the watchdog in time.",
 196         optional => 1,
 197     },
 198 };
 199 PVE::JSONSchema::register_format('pve-qm-watchdog', $watchdog_fmt);
 200
 201 my $confdesc = {
 202     onboot => {
 203         optional => 1,
 204         type => 'boolean',
 205         description => "Specifies whether a VM will be started during system bootup.",
 206         default => 0,
 207     },
 208     autostart => {
 209         optional => 1,
 210         type => 'boolean',
 211         description => "Automatic restart after crash (currently ignored).",
 212         default => 0,
 213     },
 214     hotplug => {
 215         optional => 1,
 216         type => 'string', format => 'pve-hotplug-features',
 217         description => "Selectively enable hotplug features. This is a comma separated list of hotplug features: 'network', 'disk', 'cpu', 'memory' and 'usb'. Use '0' to disable hotplug completely. Value '1' is an alias for the default 'network,disk,usb'.",
 218         default => 'network,disk,usb',
 219     },
 220     reboot => {
 221         optional => 1,
 222         type => 'boolean',
 223         description => "Allow reboot. If set to '0' the VM exit on reboot.",
 224         default => 1,
 225     },
 226     lock => {
 227         optional => 1,
 228         type => 'string',
 229         description => "Lock/unlock the VM.",
 230         enum => [qw(migrate backup snapshot rollback)],
 231     },
 232     cpulimit => {
 233         optional => 1,
 234         type => 'number',
 235         description => "Limit of CPU usage.",
 236         verbose_description => "Limit of CPU usage.\n\nNOTE: If the computer has 2 CPUs, it has total of '2' CPU time. Value '0' indicates no CPU limit.",
 237         minimum => 0,
 238         maximum => 128,
 239         default => 0,
 240     },
 241     cpuunits => {
 242         optional => 1,
 243         type => 'integer',
 244         description => "CPU weight for a VM.",
 245         verbose_description => "CPU weight for a VM. Argument is used in the kernel fair scheduler. The larger the number is, the more CPU time this VM gets. Number is relative to weights of all the other running VMs.",
 246         minimum => 2,
 247         maximum => 262144,
 248         default => 1024,
 249     },
 250     memory => {
 251         optional => 1,
 252         type => 'integer',
 253         description => "Amount of RAM for the VM in MB. This is the maximum available memory when you use the balloon device.",
 254         minimum => 16,
 255         default => 512,
 256     },
 257     balloon => {
 258         optional => 1,
 259         type => 'integer',
 260         description => "Amount of target RAM for the VM in MB. Using zero disables the ballon driver.",
 261         minimum => 0,
 262     },
 263     shares => {
 264         optional => 1,
 265         type => 'integer',
 266         description => "Amount of memory shares for auto-ballooning. The larger the number is, the more memory this VM gets. Number is relative to weights of all other running VMs. Using zero disables auto-ballooning",
 267         minimum => 0,
 268         maximum => 50000,
 269         default => 1000,
 270     },
 271     keyboard => {
 272         optional => 1,
 273         type => 'string',
 274         description => "Keybord layout for vnc server. Default is read from the '/etc/pve/datacenter.conf' configuration file.".
 275                        "It should not be necessary to set it.",
 276         enum => PVE::Tools::kvmkeymaplist(),
 277         default => undef,
 278     },
 279     name => {
 280         optional => 1,
 281         type => 'string', format => 'dns-name',
 282         description => "Set a name for the VM. Only used on the configuration web interface.",
 283     },
 284     scsihw => {
 285         optional => 1,
 286         type => 'string',
 287         description => "SCSI controller model",
 288         enum => [qw(lsi lsi53c810 virtio-scsi-pci virtio-scsi-single megasas pvscsi)],
 289         default => 'lsi',
 290     },
 291     description => {
 292         optional => 1,
 293         type => 'string',
 294         description => "Description for the VM. Only used on the configuration web interface. This is saved as comment inside the configuration file.",
 295     },
 296     ostype => {
 297         optional => 1,
 298         type => 'string',
 299         enum => [qw(other wxp w2k w2k3 w2k8 wvista win7 win8 win10 l24 l26 solaris)],
 300         description => "Specify guest operating system.",
 301         verbose_description => <<EODESC,
 302 Specify guest operating system. This is used to enable special
 303 optimization/features for specific operating systems:
 304
 305 [horizontal]
 306 other;; unspecified OS
 307 wxp;; Microsoft Windows XP
 308 w2k;; Microsoft Windows 2000
 309 w2k3;; Microsoft Windows 2003
 310 w2k8;; Microsoft Windows 2008
 311 wvista;; Microsoft Windows Vista
 312 win7;; Microsoft Windows 7
 313 win8;; Microsoft Windows 8/2012/2012r2
 314 win10;; Microsoft Windows 10/2016
 315 l24;; Linux 2.4 Kernel
 316 l26;; Linux 2.6/3.X Kernel
 317 solaris;; Solaris/OpenSolaris/OpenIndiania kernel
 318 EODESC
 319     },
 320     boot => {
 321         optional => 1,
 322         type => 'string',
 323         description => "Boot on floppy (a), hard disk (c), CD-ROM (d), or network (n).",
 324         pattern => '[acdn]{1,4}',
 325         default => 'cdn',
 326     },
 327     bootdisk => {
 328         optional => 1,
 329         type => 'string', format => 'pve-qm-bootdisk',
 330         description => "Enable booting from specified disk.",
 331         pattern => '(ide|sata|scsi|virtio)\d+',
 332     },
 333     smp => {
 334         optional => 1,
 335         type => 'integer',
 336         description => "The number of CPUs. Please use option -sockets instead.",
 337         minimum => 1,
 338         default => 1,
 339     },
 340     sockets => {
 341         optional => 1,
 342         type => 'integer',
 343         description => "The number of CPU sockets.",
 344         minimum => 1,
 345         default => 1,
 346     },
 347     cores => {
 348         optional => 1,
 349         type => 'integer',
 350         description => "The number of cores per socket.",
 351         minimum => 1,
 352         default => 1,
 353     },
 354     numa => {
 355         optional => 1,
 356         type => 'boolean',
 357         description => "Enable/disable NUMA.",
 358         default => 0,
 359     },
 360     hugepages => {
 361         optional => 1,
 362         type => 'string',
 363         description => "Enable/disable hugepages memory.",
 364         enum => [qw(any 2 1024)],
 365     },
 366     vcpus => {
 367         optional => 1,
 368         type => 'integer',
 369         description => "Number of hotplugged vcpus.",
 370         minimum => 1,
 371         default => 0,
 372     },
 373     acpi => {
 374         optional => 1,
 375         type => 'boolean',
 376         description => "Enable/disable ACPI.",
 377         default => 1,
 378     },
 379     agent => {
 380         optional => 1,
 381         type => 'boolean',
 382         description => "Enable/disable Qemu GuestAgent.",
 383         default => 0,
 384     },
 385     kvm => {
 386         optional => 1,
 387         type => 'boolean',
 388         description => "Enable/disable KVM hardware virtualization.",
 389         default => 1,
 390     },
 391     tdf => {
 392         optional => 1,
 393         type => 'boolean',
 394         description => "Enable/disable time drift fix.",
 395         default => 0,
 396     },
 397     localtime => {
 398         optional => 1,
 399         type => 'boolean',
 400         description => "Set the real time clock to local time. This is enabled by default if ostype indicates a Microsoft OS.",
 401     },
 402     freeze => {
 403         optional => 1,
 404         type => 'boolean',
 405         description => "Freeze CPU at startup (use 'c' monitor command to start execution).",
 406     },
 407     vga => {
 408         optional => 1,
 409         type => 'string',
 410         description => "Select the VGA type.",
 411         verbose_description => "Select the VGA type. If you want to use high resolution" .
 412             " modes (>= 1280x1024x16) then you should use the options " .
 413             "'std' or 'vmware'. Default is 'std' for win8/win7/w2k8, and " .
 414             "'cirrus' for other OS types. The 'qxl' option enables the SPICE " .
 415             "display sever. For win* OS you can select how many independent " .
 416             "displays you want, Linux guests can add displays them self. " .
 417             "You can also run without any graphic card, using a serial device" .
 418             " as terminal.",
 419         enum => [qw(std cirrus vmware qxl serial0 serial1 serial2 serial3 qxl2 qxl3 qxl4)],
 420     },
 421     watchdog => {
 422         optional => 1,
 423         type => 'string', format => 'pve-qm-watchdog',
 424         description => "Create a virtual hardware watchdog device.",
 425         verbose_description => "Create a virtual hardware watchdog device. Once enabled" .
 426             " (by a guest action), the watchdog must be periodically polled " .
 427             "by an agent inside the guest or else the watchdog will reset " .
 428             "the guest (or execute the respective action specified)",
 429     },
 430     startdate => {
 431         optional => 1,
 432         type => 'string',
 433         typetext => "(now | YYYY-MM-DD | YYYY-MM-DDTHH:MM:SS)",
 434         description => "Set the initial date of the real time clock. Valid format for date are: 'now' or '2006-06-17T16:01:21' or '2006-06-17'.",
 435         pattern => '(now|\d{4}-\d{1,2}-\d{1,2}(T\d{1,2}:\d{1,2}:\d{1,2})?)',
 436         default => 'now',
 437     },
 438     startup =>  get_standard_option('pve-startup-order'),
 439     template => {
 440         optional => 1,
 441         type => 'boolean',
 442         description => "Enable/disable Template.",
 443         default => 0,
 444     },
 445     args => {
 446         optional => 1,
 447         type => 'string',
 448         description => "Arbitrary arguments passed to kvm.",
 449         verbose_description => <<EODESCR,
 450 Arbitrary arguments passed to kvm, for example:
 451
 452 args: -no-reboot -no-hpet
 453
 454 NOTE: this option is for experts only.
 455 EODESCR
 456     },
 457     tablet => {
 458         optional => 1,
 459         type => 'boolean',
 460         default => 1,
 461         description => "Enable/disable the USB tablet device.",
 462         verbose_description => "Enable/disable the USB tablet device. This device is " .
 463             "usually needed to allow absolute mouse positioning with VNC. " .
 464             "Else the mouse runs out of sync with normal VNC clients. " .
 465             "If you're running lots of console-only guests on one host, " .
 466             "you may consider disabling this to save some context switches. " .
 467             "This is turned off by default if you use spice (-vga=qxl).",
 468     },
 469     migrate_speed => {
 470         optional => 1,
 471         type => 'integer',
 472         description => "Set maximum speed (in MB/s) for migrations. Value 0 is no limit.",
 473         minimum => 0,
 474         default => 0,
 475     },
 476     migrate_downtime => {
 477         optional => 1,
 478         type => 'number',
 479         description => "Set maximum tolerated downtime (in seconds) for migrations.",
 480         minimum => 0,
 481         default => 0.1,
 482     },
 483     cdrom => {
 484         optional => 1,
 485         type => 'string', format => 'pve-qm-ide',
 486         typetext => '<volume>',
 487         description => "This is an alias for option -ide2",
 488     },
 489     cpu => {
 490         optional => 1,
 491         description => "Emulated CPU type.",
 492         type => 'string',
 493         format => $cpu_fmt,
 494     },
 495     parent => get_standard_option('pve-snapshot-name', {
 496         optional => 1,
 497         description => "Parent snapshot name. This is used internally, and should not be modified.",
 498     }),
 499     snaptime => {
 500         optional => 1,
 501         description => "Timestamp for snapshots.",
 502         type => 'integer',
 503         minimum => 0,
 504     },
 505     vmstate => {
 506         optional => 1,
 507         type => 'string', format => 'pve-volume-id',
 508         description => "Reference to a volume which stores the VM state. This is used internally for snapshots.",
 509     },
 510     vmstatestorage => get_standard_option('pve-storage-id', {
 511         description => "Default storage for VM state volumes/files.",
 512         optional => 1,
 513     }),
 514     machine => {
 515         description => "Specific the Qemu machine type.",
 516         type => 'string',
 517         pattern => '(pc|pc(-i440fx)?-\d+\.\d+(\.pxe)?|q35|pc-q35-\d+\.\d+(\.pxe)?)',
 518         maxLength => 40,
 519         optional => 1,
 520     },
 521     smbios1 => {
 522         description => "Specify SMBIOS type 1 fields.",
 523         type => 'string', format => 'pve-qm-smbios1',
 524         maxLength => 256,
 525         optional => 1,
 526     },
 527     protection => {
 528         optional => 1,
 529         type => 'boolean',
 530         description => "Sets the protection flag of the VM. This will disable the remove VM and remove disk operations.",
 531         default => 0,
 532     },
 533     bios => {
 534         optional => 1,
 535         type => 'string',
 536         enum => [ qw(seabios ovmf) ],
 537         description => "Select BIOS implementation.",
 538         default => 'seabios',
 539     },
 540 };
 541
 542 my $confdesc_cloudinit = {
 543     citype => {
 544         optional => 1,
 545         type => 'string',
 546         description => 'Specifies the cloud-init configuration format. The default depends on the configured operating system type (`ostype`. We use the `nocloud` format for Linux, and `configdrive2` for windows.',
 547         enum => ['configdrive2', 'nocloud'],
 548     },
 549     ciuser => {
 550         optional => 1,
 551         type => 'string',
 552         description => "cloud-init: User name to change ssh keys and password for instead of the image's configured default user.",
 553     },
 554     cipassword => {
 555         optional => 1,
 556         type => 'string',
 557         description => 'cloud-init: Password to assign the user. Using this is generally not recommended. Use ssh keys instead. Also note that older cloud-init versions do not support hashed passwords.',
 558     },
 559     searchdomain => {
 560         optional => 1,
 561         type => 'string',
 562         description => "cloud-init: Sets DNS search domains for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 563     },
 564     nameserver => {
 565         optional => 1,
 566         type => 'string', format => 'address-list',
 567         description => "cloud-init: Sets DNS server IP address for a container. Create will automatically use the setting from the host if neither searchdomain nor nameserver are set.",
 568     },
 569     sshkeys => {
 570         optional => 1,
 571         type => 'string',
 572         format => 'urlencoded',
 573         description => "cloud-init: Setup public SSH keys (one key per line, OpenSSH format).",
 574     },
 575 };
 576
 577 # what about other qemu settings ?
 578 #cpu => 'string',
 579 #machine => 'string',
 580 #fda => 'file',
 581 #fdb => 'file',
 582 #mtdblock => 'file',
 583 #sd => 'file',
 584 #pflash => 'file',
 585 #snapshot => 'bool',
 586 #bootp => 'file',
 587 ##tftp => 'dir',
 588 ##smb => 'dir',
 589 #kernel => 'file',
 590 #append => 'string',
 591 #initrd => 'file',
 592 ##soundhw => 'string',
 593
 594 while (my ($k, $v) = each %$confdesc) {
 595     PVE::JSONSchema::register_standard_option("pve-qm-$k", $v);
 596 }
 597
 598 my $MAX_IDE_DISKS = 4;
 599 my $MAX_SCSI_DISKS = 14;
 600 my $MAX_VIRTIO_DISKS = 16;
 601 my $MAX_SATA_DISKS = 6;
 602 my $MAX_USB_DEVICES = 5;
 603 my $MAX_NETS = 32;
 604 my $MAX_UNUSED_DISKS = 8;
 605 my $MAX_HOSTPCI_DEVICES = 4;
 606 my $MAX_SERIAL_PORTS = 4;
 607 my $MAX_PARALLEL_PORTS = 3;
 608 my $MAX_NUMA = 8;
 609
 610 my $numa_fmt = {
 611     cpus => {
 612         type => "string",
 613         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 614         description => "CPUs accessing this NUMA node.",
 615         format_description => "id[-id];...",
 616     },
 617     memory => {
 618         type => "number",
 619         description => "Amount of memory this NUMA node provides.",
 620         optional => 1,
 621     },
 622     hostnodes => {
 623         type => "string",
 624         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 625         description => "Host NUMA nodes to use.",
 626         format_description => "id[-id];...",
 627         optional => 1,
 628     },
 629     policy => {
 630         type => 'string',
 631         enum => [qw(preferred bind interleave)],
 632         description => "NUMA allocation policy.",
 633         optional => 1,
 634     },
 635 };
 636 PVE::JSONSchema::register_format('pve-qm-numanode', $numa_fmt);
 637 my $numadesc = {
 638     optional => 1,
 639     type => 'string', format => $numa_fmt,
 640     description => "NUMA topology.",
 641 };
 642 PVE::JSONSchema::register_standard_option("pve-qm-numanode", $numadesc);
 643
 644 for (my $i = 0; $i < $MAX_NUMA; $i++)  {
 645     $confdesc->{"numa$i"} = $numadesc;
 646 }
 647
 648 my $nic_model_list = ['rtl8139', 'ne2k_pci', 'e1000',  'pcnet',  'virtio',
 649                       'ne2k_isa', 'i82551', 'i82557b', 'i82559er', 'vmxnet3',
 650                       'e1000-82540em', 'e1000-82544gc', 'e1000-82545em'];
 651 my $nic_model_list_txt = join(' ', sort @$nic_model_list);
 652
 653 my $net_fmt_bridge_descr = <<__EOD__;
 654 Bridge to attach the network device to. The Proxmox VE standard bridge
 655 is called 'vmbr0'.
 656
 657 If you do not specify a bridge, we create a kvm user (NATed) network
 658 device, which provides DHCP and DNS services. The following addresses
 659 are used:
 660
 661  10.0.2.2   Gateway
 662  10.0.2.3   DNS Server
 663  10.0.2.4   SMB Server
 664
 665 The DHCP server assign addresses to the guest starting from 10.0.2.15.
 666 __EOD__
 667
 668 my $net_fmt = {
 669     macaddr => {
 670         type => 'string',
 671         pattern => qr/[0-9a-f]{2}(?::[0-9a-f]{2}){5}/i,
 672         description => "MAC address. That address must be unique withing your network. This is automatically generated if not specified.",
 673         format_description => "XX:XX:XX:XX:XX:XX",
 674         optional => 1,
 675     },
 676     model => {
 677         type => 'string',
 678         description => "Network Card Model. The 'virtio' model provides the best performance with very low CPU overhead. If your guest does not support this driver, it is usually best to use 'e1000'.",
 679         enum => $nic_model_list,
 680         default_key => 1,
 681     },
 682     (map { $_ => { keyAlias => 'model', alias => 'macaddr' }} @$nic_model_list),
 683     bridge => {
 684         type => 'string',
 685         description => $net_fmt_bridge_descr,
 686         format_description => 'bridge',
 687         optional => 1,
 688     },
 689     queues => {
 690         type => 'integer',
 691         minimum => 0, maximum => 16,
 692         description => 'Number of packet queues to be used on the device.',
 693         optional => 1,
 694     },
 695     rate => {
 696         type => 'number',
 697         minimum => 0,
 698         description => "Rate limit in mbps (megabytes per second) as floating point number.",
 699         optional => 1,
 700     },
 701     tag => {
 702         type => 'integer',
 703         minimum => 1, maximum => 4094,
 704         description => 'VLAN tag to apply to packets on this interface.',
 705         optional => 1,
 706     },
 707     trunks => {
 708         type => 'string',
 709         pattern => qr/\d+(?:-\d+)?(?:;\d+(?:-\d+)?)*/,
 710         description => 'VLAN trunks to pass through this interface.',
 711         format_description => 'vlanid[;vlanid...]',
 712         optional => 1,
 713     },
 714     firewall => {
 715         type => 'boolean',
 716         description => 'Whether this interface should be protected by the firewall.',
 717         optional => 1,
 718     },
 719     link_down => {
 720         type => 'boolean',
 721         description => 'Whether this interface should be disconnected (like pulling the plug).',
 722         optional => 1,
 723     },
 724 };
 725
 726 my $netdesc = {
 727     optional => 1,
 728     type => 'string', format => $net_fmt,
 729     description => "Specify network devices.",
 730 };
 731
 732 PVE::JSONSchema::register_standard_option("pve-qm-net", $netdesc);
 733
 734 my $ipconfig_fmt = {
 735     ip => {
 736         type => 'string',
 737         format => 'pve-ipv4-config',
 738         format_description => 'IPv4Format/CIDR',
 739         description => 'IPv4 address in CIDR format.',
 740         optional => 1,
 741         default => 'dhcp',
 742     },
 743     gw => {
 744         type => 'string',
 745         format => 'ipv4',
 746         format_description => 'GatewayIPv4',
 747         description => 'Default gateway for IPv4 traffic.',
 748         optional => 1,
 749         requires => 'ip',
 750     },
 751     ip6 => {
 752         type => 'string',
 753         format => 'pve-ipv6-config',
 754         format_description => 'IPv6Format/CIDR',
 755         description => 'IPv6 address in CIDR format.',
 756         optional => 1,
 757         default => 'dhcp',
 758     },
 759     gw6 => {
 760         type => 'string',
 761         format => 'ipv6',
 762         format_description => 'GatewayIPv6',
 763         description => 'Default gateway for IPv6 traffic.',
 764         optional => 1,
 765         requires => 'ip6',
 766     },
 767 };
 768 PVE::JSONSchema::register_format('pve-qm-ipconfig', $ipconfig_fmt);
 769 my $ipconfigdesc = {
 770     optional => 1,
 771     type => 'string', format => 'pve-qm-ipconfig',
 772     description => <<'EODESCR',
 773 cloud-init: Specify IP addresses and gateways for the corresponding interface.
 774
 775 IP addresses use CIDR notation, gateways are optional but need an IP of the same type specified.
 776
 777 The special string 'dhcp' can be used for IP addresses to use DHCP, in which case no explicit gateway should be provided.
 778 For IPv6 the special string 'auto' can be used to use stateless autoconfiguration.
 779
 780 If cloud-init is enabled and neither an IPv4 nor an IPv6 address is specified, it defaults to using dhcp on IPv4.
 781 EODESCR
 782 };
 783 PVE::JSONSchema::register_standard_option("pve-qm-ipconfig", $netdesc);
 784
 785 for (my $i = 0; $i < $MAX_NETS; $i++)  {
 786     $confdesc->{"net$i"} = $netdesc;
 787     $confdesc_cloudinit->{"ipconfig$i"} = $ipconfigdesc;
 788 }
 789
 790 foreach my $key (keys %$confdesc_cloudinit) {
 791     $confdesc->{$key} = $confdesc_cloudinit->{$key};
 792 }
 793
 794 PVE::JSONSchema::register_format('pve-volume-id-or-qm-path', \&verify_volume_id_or_qm_path);
 795 sub verify_volume_id_or_qm_path {
 796     my ($volid, $noerr) = @_;
 797
 798     if ($volid eq 'none' || $volid eq 'cdrom' || $volid =~ m|^/|) {
 799         return $volid;
 800     }
 801
 802     # if its neither 'none' nor 'cdrom' nor a path, check if its a volume-id
 803     $volid = eval { PVE::JSONSchema::check_format('pve-volume-id', $volid, '') };
 804     if ($@) {
 805         return undef if $noerr;
 806         die $@;
 807     }
 808     return $volid;
 809 }
 810
 811 my $drivename_hash;
 812
 813 my %drivedesc_base = (
 814     volume => { alias => 'file' },
 815     file => {
 816         type => 'string',
 817         format => 'pve-volume-id-or-qm-path',
 818         default_key => 1,
 819         format_description => 'volume',
 820         description => "The drive's backing volume.",
 821     },
 822     media => {
 823         type => 'string',
 824         enum => [qw(cdrom disk)],
 825         description => "The drive's media type.",
 826         default => 'disk',
 827         optional => 1
 828     },
 829     cyls => {
 830         type => 'integer',
 831         description => "Force the drive's physical geometry to have a specific cylinder count.",
 832         optional => 1
 833     },
 834     heads => {
 835         type => 'integer',
 836         description => "Force the drive's physical geometry to have a specific head count.",
 837         optional => 1
 838     },
 839     secs => {
 840         type => 'integer',
 841         description => "Force the drive's physical geometry to have a specific sector count.",
 842         optional => 1
 843     },
 844     trans => {
 845         type => 'string',
 846         enum => [qw(none lba auto)],
 847         description => "Force disk geometry bios translation mode.",
 848         optional => 1,
 849     },
 850     snapshot => {
 851         type => 'boolean',
 852         description => "Controls qemu's snapshot mode feature."
 853             . " If activated, changes made to the disk are temporary and will"
 854             . " be discarded when the VM is shutdown.",
 855         optional => 1,
 856     },
 857     cache => {
 858         type => 'string',
 859         enum => [qw(none writethrough writeback unsafe directsync)],
 860         description => "The drive's cache mode",
 861         optional => 1,
 862     },
 863     format => get_standard_option('pve-qm-image-format'),
 864     size => {
 865         type => 'string',
 866         format => 'disk-size',
 867         format_description => 'DiskSize',
 868         description => "Disk size. This is purely informational and has no effect.",
 869         optional => 1,
 870     },
 871     backup => {
 872         type => 'boolean',
 873         description => "Whether the drive should be included when making backups.",
 874         optional => 1,
 875     },
 876     replicate => {
 877         type => 'boolean',
 878         description => 'Whether the drive should considered for replication jobs.',
 879         optional => 1,
 880         default => 1,
 881     },
 882     rerror => {
 883         type => 'string',
 884         enum => [qw(ignore report stop)],
 885         description => 'Read error action.',
 886         optional => 1,
 887     },
 888     werror => {
 889         type => 'string',
 890         enum => [qw(enospc ignore report stop)],
 891         description => 'Write error action.',
 892         optional => 1,
 893     },
 894     aio => {
 895         type => 'string',
 896         enum => [qw(native threads)],
 897         description => 'AIO type to use.',
 898         optional => 1,
 899     },
 900     discard => {
 901         type => 'string',
 902         enum => [qw(ignore on)],
 903         description => 'Controls whether to pass discard/trim requests to the underlying storage.',
 904         optional => 1,
 905     },
 906     detect_zeroes => {
 907         type => 'boolean',
 908         description => 'Controls whether to detect and try to optimize writes of zeroes.',
 909         optional => 1,
 910     },
 911     serial => {
 912         type => 'string',
 913         format => 'urlencoded',
 914         format_description => 'serial',
 915         maxLength => 20*3, # *3 since it's %xx url enoded
 916         description => "The drive's reported serial number, url-encoded, up to 20 bytes long.",
 917         optional => 1,
 918     },
 919     shared => {
 920         type => 'boolean',
 921         description => 'Mark this locally-managed volume as available on all nodes',
 922         verbose_description => "Mark this locally-managed volume as available on all nodes.\n\nWARNING: This option does not share the volume automatically, it assumes it is shared already!",
 923         optional => 1,
 924         default => 0,
 925     }
 926 );
 927
 928 my %iothread_fmt = ( iothread => {
 929         type => 'boolean',
 930         description => "Whether to use iothreads for this drive",
 931         optional => 1,
 932 });
 933
 934 my %model_fmt = (
 935     model => {
 936         type => 'string',
 937         format => 'urlencoded',
 938         format_description => 'model',
 939         maxLength => 40*3, # *3 since it's %xx url enoded
 940         description => "The drive's reported model name, url-encoded, up to 40 bytes long.",
 941         optional => 1,
 942     },
 943 );
 944
 945 my %queues_fmt = (
 946     queues => {
 947         type => 'integer',
 948         description => "Number of queues.",
 949         minimum => 2,
 950         optional => 1
 951     }
 952 );
 953
 954 my %scsiblock_fmt = (
 955     scsiblock => {
 956         type => 'boolean',
 957         description => "whether to use scsi-block for full passthrough of host block device\n\nWARNING: can lead to I/O errors in combination with low memory or high memory fragmentation on host",
 958         optional => 1,
 959         default => 0,
 960     },
 961 );
 962
 963 my $add_throttle_desc = sub {
 964     my ($key, $type, $what, $unit, $longunit, $minimum) = @_;
 965     my $d = {
 966         type => $type,
 967         format_description => $unit,
 968         description => "Maximum $what in $longunit.",
 969         optional => 1,
 970     };
 971     $d->{minimum} = $minimum if defined($minimum);
 972     $drivedesc_base{$key} = $d;
 973 };
 974 # throughput: (leaky bucket)
 975 $add_throttle_desc->('bps',     'integer', 'r/w speed',   'bps',  'bytes per second');
 976 $add_throttle_desc->('bps_rd',  'integer', 'read speed',  'bps',  'bytes per second');
 977 $add_throttle_desc->('bps_wr',  'integer', 'write speed', 'bps',  'bytes per second');
 978 $add_throttle_desc->('mbps',    'number',  'r/w speed',   'mbps', 'megabytes per second');
 979 $add_throttle_desc->('mbps_rd', 'number',  'read speed',  'mbps', 'megabytes per second');
 980 $add_throttle_desc->('mbps_wr', 'number',  'write speed', 'mbps', 'megabytes per second');
 981 $add_throttle_desc->('iops',    'integer', 'r/w I/O',     'iops', 'operations per second');
 982 $add_throttle_desc->('iops_rd', 'integer', 'read I/O',    'iops', 'operations per second');
 983 $add_throttle_desc->('iops_wr', 'integer', 'write I/O',   'iops', 'operations per second');
 984
 985 # pools: (pool of IO before throttling starts taking effect)
 986 $add_throttle_desc->('mbps_max',    'number',  'unthrottled r/w pool',       'mbps', 'megabytes per second');
 987 $add_throttle_desc->('mbps_rd_max', 'number',  'unthrottled read pool',      'mbps', 'megabytes per second');
 988 $add_throttle_desc->('mbps_wr_max', 'number',  'unthrottled write pool',     'mbps', 'megabytes per second');
 989 $add_throttle_desc->('iops_max',    'integer', 'unthrottled r/w I/O pool',   'iops', 'operations per second');
 990 $add_throttle_desc->('iops_rd_max', 'integer', 'unthrottled read I/O pool',  'iops', 'operations per second');
 991 $add_throttle_desc->('iops_wr_max', 'integer', 'unthrottled write I/O pool', 'iops', 'operations per second');
 992
 993 # burst lengths
 994 $add_throttle_desc->('bps_max_length',     'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 995 $add_throttle_desc->('bps_rd_max_length',  'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 996 $add_throttle_desc->('bps_wr_max_length',  'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
 997 $add_throttle_desc->('iops_max_length',    'integer', 'length of I/O bursts',       'seconds', 'seconds', 1);
 998 $add_throttle_desc->('iops_rd_max_length', 'integer', 'length of read I/O bursts',  'seconds', 'seconds', 1);
 999 $add_throttle_desc->('iops_wr_max_length', 'integer', 'length of write I/O bursts', 'seconds', 'seconds', 1);
1000
1001 # legacy support
1002 $drivedesc_base{'bps_rd_length'} = { alias => 'bps_rd_max_length' };
1003 $drivedesc_base{'bps_wr_length'} = { alias => 'bps_wr_max_length' };
1004 $drivedesc_base{'iops_rd_length'} = { alias => 'iops_rd_max_length' };
1005 $drivedesc_base{'iops_wr_length'} = { alias => 'iops_wr_max_length' };
1006
1007 my $ide_fmt = {
1008     %drivedesc_base,
1009     %model_fmt,
1010 };
1011 PVE::JSONSchema::register_format("pve-qm-ide", $ide_fmt);
1012
1013 my $idedesc = {
1014     optional => 1,
1015     type => 'string', format => $ide_fmt,
1016     description => "Use volume as IDE hard disk or CD-ROM (n is 0 to " .($MAX_IDE_DISKS -1) . ").",
1017 };
1018 PVE::JSONSchema::register_standard_option("pve-qm-ide", $idedesc);
1019
1020 my $scsi_fmt = {
1021     %drivedesc_base,
1022     %iothread_fmt,
1023     %queues_fmt,
1024     %scsiblock_fmt,
1025 };
1026 my $scsidesc = {
1027     optional => 1,
1028     type => 'string', format => $scsi_fmt,
1029     description => "Use volume as SCSI hard disk or CD-ROM (n is 0 to " . ($MAX_SCSI_DISKS - 1) . ").",
1030 };
1031 PVE::JSONSchema::register_standard_option("pve-qm-scsi", $scsidesc);
1032
1033 my $sata_fmt = {
1034     %drivedesc_base,
1035 };
1036 my $satadesc = {
1037     optional => 1,
1038     type => 'string', format => $sata_fmt,
1039     description => "Use volume as SATA hard disk or CD-ROM (n is 0 to " . ($MAX_SATA_DISKS - 1). ").",
1040 };
1041 PVE::JSONSchema::register_standard_option("pve-qm-sata", $satadesc);
1042
1043 my $virtio_fmt = {
1044     %drivedesc_base,
1045     %iothread_fmt,
1046 };
1047 my $virtiodesc = {
1048     optional => 1,
1049     type => 'string', format => $virtio_fmt,
1050     description => "Use volume as VIRTIO hard disk (n is 0 to " . ($MAX_VIRTIO_DISKS - 1) . ").",
1051 };
1052 PVE::JSONSchema::register_standard_option("pve-qm-virtio", $virtiodesc);
1053
1054 my $alldrive_fmt = {
1055     %drivedesc_base,
1056     %iothread_fmt,
1057     %model_fmt,
1058     %queues_fmt,
1059     %scsiblock_fmt,
1060 };
1061
1062 my $efidisk_fmt = {
1063     volume => { alias => 'file' },
1064     file => {
1065         type => 'string',
1066         format => 'pve-volume-id-or-qm-path',
1067         default_key => 1,
1068         format_description => 'volume',
1069         description => "The drive's backing volume.",
1070     },
1071     format => get_standard_option('pve-qm-image-format'),
1072     size => {
1073         type => 'string',
1074         format => 'disk-size',
1075         format_description => 'DiskSize',
1076         description => "Disk size. This is purely informational and has no effect.",
1077         optional => 1,
1078     },
1079 };
1080
1081 my $efidisk_desc = {
1082     optional => 1,
1083     type => 'string', format => $efidisk_fmt,
1084     description => "Configure a Disk for storing EFI vars",
1085 };
1086
1087 PVE::JSONSchema::register_standard_option("pve-qm-efidisk", $efidisk_desc);
1088
1089 my $usb_fmt = {
1090     host => {
1091         default_key => 1,
1092         type => 'string', format => 'pve-qm-usb-device',
1093         format_description => 'HOSTUSBDEVICE|spice',
1094         description => <<EODESCR,
1095 The Host USB device or port or the value 'spice'. HOSTUSBDEVICE syntax is:
1096
1097  'bus-port(.port)*' (decimal numbers) or
1098  'vendor_id:product_id' (hexadeciaml numbers) or
1099  'spice'
1100
1101 You can use the 'lsusb -t' command to list existing usb devices.
1102
1103 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1104
1105 The value 'spice' can be used to add a usb redirection devices for spice.
1106 EODESCR
1107     },
1108     usb3 => {
1109         optional => 1,
1110         type => 'boolean',
1111         description => "Specifies whether if given host option is a USB3 device or port (this does currently not work reliably with spice redirection and is then ignored).",
1112         default => 0,
1113     },
1114 };
1115
1116 my $usbdesc = {
1117     optional => 1,
1118     type => 'string', format => $usb_fmt,
1119     description => "Configure an USB device (n is 0 to 4).",
1120 };
1121 PVE::JSONSchema::register_standard_option("pve-qm-usb", $usbdesc);
1122
1123 # NOTE: the match-groups of this regex are used in parse_hostpci
1124 my $PCIRE = qr/([a-f0-9]{2}:[a-f0-9]{2})(?:\.([a-f0-9]))?/;
1125 my $hostpci_fmt = {
1126     host => {
1127         default_key => 1,
1128         type => 'string',
1129         pattern => qr/$PCIRE(;$PCIRE)*/,
1130         format_description => 'HOSTPCIID[;HOSTPCIID2...]',
1131         description => <<EODESCR,
1132 Host PCI device pass through. The PCI ID of a host's PCI device or a list
1133 of PCI virtual functions of the host. HOSTPCIID syntax is:
1134
1135 'bus:dev.func' (hexadecimal numbers)
1136
1137 You can us the 'lspci' command to list existing PCI devices.
1138 EODESCR
1139     },
1140     rombar => {
1141         type => 'boolean',
1142         description =>  "Specify whether or not the device's ROM will be visible in the guest's memory map.",
1143         optional => 1,
1144         default => 1,
1145     },
1146     romfile => {
1147         type => 'string',
1148         pattern => '[^,;]+',
1149         format_description => 'string',
1150         description => "Custom pci device rom filename (must be located in /usr/share/kvm/).",
1151         optional => 1,
1152     },
1153     pcie => {
1154         type => 'boolean',
1155         description =>  "Choose the PCI-express bus (needs the 'q35' machine model).",
1156         optional => 1,
1157         default => 0,
1158     },
1159     'x-vga' => {
1160         type => 'boolean',
1161         description =>  "Enable vfio-vga device support.",
1162         optional => 1,
1163         default => 0,
1164     },
1165 };
1166 PVE::JSONSchema::register_format('pve-qm-hostpci', $hostpci_fmt);
1167
1168 my $hostpcidesc = {
1169         optional => 1,
1170         type => 'string', format => 'pve-qm-hostpci',
1171         description => "Map host PCI devices into guest.",
1172         verbose_description =>  <<EODESCR,
1173 Map host PCI devices into guest.
1174
1175 NOTE: This option allows direct access to host hardware. So it is no longer
1176 possible to migrate such machines - use with special care.
1177
1178 CAUTION: Experimental! User reported problems with this option.
1179 EODESCR
1180 };
1181 PVE::JSONSchema::register_standard_option("pve-qm-hostpci", $hostpcidesc);
1182
1183 my $serialdesc = {
1184         optional => 1,
1185         type => 'string',
1186         pattern => '(/dev/.+|socket)',
1187         description =>  "Create a serial device inside the VM (n is 0 to 3)",
1188         verbose_description =>  <<EODESCR,
1189 Create a serial device inside the VM (n is 0 to 3), and pass through a
1190 host serial device (i.e. /dev/ttyS0), or create a unix socket on the
1191 host side (use 'qm terminal' to open a terminal connection).
1192
1193 NOTE: If you pass through a host serial device, it is no longer possible to migrate such machines - use with special care.
1194
1195 CAUTION: Experimental! User reported problems with this option.
1196 EODESCR
1197 };
1198
1199 my $paralleldesc= {
1200         optional => 1,
1201         type => 'string',
1202         pattern => '/dev/parport\d+|/dev/usb/lp\d+',
1203         description =>  "Map host parallel devices (n is 0 to 2).",
1204         verbose_description =>  <<EODESCR,
1205 Map host parallel devices (n is 0 to 2).
1206
1207 NOTE: This option allows direct access to host hardware. So it is no longer possible to migrate such machines - use with special care.
1208
1209 CAUTION: Experimental! User reported problems with this option.
1210 EODESCR
1211 };
1212
1213 for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
1214     $confdesc->{"parallel$i"} = $paralleldesc;
1215 }
1216
1217 for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
1218     $confdesc->{"serial$i"} = $serialdesc;
1219 }
1220
1221 for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
1222     $confdesc->{"hostpci$i"} = $hostpcidesc;
1223 }
1224
1225 for (my $i = 0; $i < $MAX_IDE_DISKS; $i++)  {
1226     $drivename_hash->{"ide$i"} = 1;
1227     $confdesc->{"ide$i"} = $idedesc;
1228 }
1229
1230 for (my $i = 0; $i < $MAX_SATA_DISKS; $i++)  {
1231     $drivename_hash->{"sata$i"} = 1;
1232     $confdesc->{"sata$i"} = $satadesc;
1233 }
1234
1235 for (my $i = 0; $i < $MAX_SCSI_DISKS; $i++)  {
1236     $drivename_hash->{"scsi$i"} = 1;
1237     $confdesc->{"scsi$i"} = $scsidesc ;
1238 }
1239
1240 for (my $i = 0; $i < $MAX_VIRTIO_DISKS; $i++)  {
1241     $drivename_hash->{"virtio$i"} = 1;
1242     $confdesc->{"virtio$i"} = $virtiodesc;
1243 }
1244
1245 $drivename_hash->{efidisk0} = 1;
1246 $confdesc->{efidisk0} = $efidisk_desc;
1247
1248 for (my $i = 0; $i < $MAX_USB_DEVICES; $i++)  {
1249     $confdesc->{"usb$i"} = $usbdesc;
1250 }
1251
1252 my $unuseddesc = {
1253     optional => 1,
1254     type => 'string', format => 'pve-volume-id',
1255     description => "Reference to unused volumes. This is used internally, and should not be modified manually.",
1256 };
1257
1258 for (my $i = 0; $i < $MAX_UNUSED_DISKS; $i++)  {
1259     $confdesc->{"unused$i"} = $unuseddesc;
1260 }
1261
1262 my $kvm_api_version = 0;
1263
1264 sub kvm_version {
1265
1266     return $kvm_api_version if $kvm_api_version;
1267
1268     my $fh = IO::File->new("</dev/kvm") ||
1269         return 0;
1270
1271     if (my $v = $fh->ioctl(KVM_GET_API_VERSION(), 0)) {
1272         $kvm_api_version = $v;
1273     }
1274
1275     $fh->close();
1276
1277     return  $kvm_api_version;
1278 }
1279
1280 my $kvm_user_version;
1281
1282 sub kvm_user_version {
1283
1284     return $kvm_user_version if $kvm_user_version;
1285
1286     $kvm_user_version = 'unknown';
1287
1288     my $code = sub {
1289         my $line = shift;
1290         if ($line =~ m/^QEMU( PC)? emulator version (\d+\.\d+(\.\d+)?)(\.\d+)?[,\s]/) {
1291             $kvm_user_version = $2;
1292         }
1293     };
1294
1295     eval { run_command("kvm -version", outfunc => $code); };
1296     warn $@ if $@;
1297
1298     return $kvm_user_version;
1299
1300 }
1301
1302 my $kernel_has_vhost_net = -c '/dev/vhost-net';
1303
1304 sub valid_drive_names {
1305     # order is important - used to autoselect boot disk
1306     return ((map { "ide$_" } (0 .. ($MAX_IDE_DISKS - 1))),
1307             (map { "scsi$_" } (0 .. ($MAX_SCSI_DISKS - 1))),
1308             (map { "virtio$_" } (0 .. ($MAX_VIRTIO_DISKS - 1))),
1309             (map { "sata$_" } (0 .. ($MAX_SATA_DISKS - 1))),
1310             'efidisk0');
1311 }
1312
1313 sub is_valid_drivename {
1314     my $dev = shift;
1315
1316     return defined($drivename_hash->{$dev});
1317 }
1318
1319 sub option_exists {
1320     my $key = shift;
1321     return defined($confdesc->{$key});
1322 }
1323
1324 sub nic_models {
1325     return $nic_model_list;
1326 }
1327
1328 sub os_list_description {
1329
1330     return {
1331         other => 'Other',
1332         wxp => 'Windows XP',
1333         w2k => 'Windows 2000',
1334         w2k3 =>, 'Windows 2003',
1335         w2k8 => 'Windows 2008',
1336         wvista => 'Windows Vista',
1337         win7 => 'Windows 7',
1338         win8 => 'Windows 8/2012',
1339         win10 => 'Windows 10/2016',
1340         l24 => 'Linux 2.4',
1341         l26 => 'Linux 2.6',
1342     };
1343 }
1344
1345 my $cdrom_path;
1346
1347 sub get_cdrom_path {
1348
1349     return  $cdrom_path if $cdrom_path;
1350
1351     return $cdrom_path = "/dev/cdrom" if -l "/dev/cdrom";
1352     return $cdrom_path = "/dev/cdrom1" if -l "/dev/cdrom1";
1353     return $cdrom_path = "/dev/cdrom2" if -l "/dev/cdrom2";
1354 }
1355
1356 sub get_iso_path {
1357     my ($storecfg, $vmid, $cdrom) = @_;
1358
1359     if ($cdrom eq 'cdrom') {
1360         return get_cdrom_path();
1361     } elsif ($cdrom eq 'none') {
1362         return '';
1363     } elsif ($cdrom =~ m|^/|) {
1364         return $cdrom;
1365     } else {
1366         return PVE::Storage::path($storecfg, $cdrom);
1367     }
1368 }
1369
1370 # try to convert old style file names to volume IDs
1371 sub filename_to_volume_id {
1372     my ($vmid, $file, $media) = @_;
1373
1374      if (!($file eq 'none' || $file eq 'cdrom' ||
1375           $file =~ m|^/dev/.+| || $file =~ m/^([^:]+):(.+)$/)) {
1376
1377         return undef if $file =~ m|/|;
1378
1379         if ($media && $media eq 'cdrom') {
1380             $file = "local:iso/$file";
1381         } else {
1382             $file = "local:$vmid/$file";
1383         }
1384     }
1385
1386     return $file;
1387 }
1388
1389 sub verify_media_type {
1390     my ($opt, $vtype, $media) = @_;
1391
1392     return if !$media;
1393
1394     my $etype;
1395     if ($media eq 'disk') {
1396         $etype = 'images';
1397     } elsif ($media eq 'cdrom') {
1398         $etype = 'iso';
1399     } else {
1400         die "internal error";
1401     }
1402
1403     return if ($vtype eq $etype);
1404
1405     raise_param_exc({ $opt => "unexpected media type ($vtype != $etype)" });
1406 }
1407
1408 sub cleanup_drive_path {
1409     my ($opt, $storecfg, $drive) = @_;
1410
1411     # try to convert filesystem paths to volume IDs
1412
1413     if (($drive->{file} !~ m/^(cdrom|none)$/) &&
1414         ($drive->{file} !~ m|^/dev/.+|) &&
1415         ($drive->{file} !~ m/^([^:]+):(.+)$/) &&
1416         ($drive->{file} !~ m/^\d+$/)) {
1417         my ($vtype, $volid) = PVE::Storage::path_to_volume_id($storecfg, $drive->{file});
1418         raise_param_exc({ $opt => "unable to associate path '$drive->{file}' to any storage"}) if !$vtype;
1419         $drive->{media} = 'cdrom' if !$drive->{media} && $vtype eq 'iso';
1420         verify_media_type($opt, $vtype, $drive->{media});
1421         $drive->{file} = $volid;
1422     }
1423
1424     $drive->{media} = 'cdrom' if !$drive->{media} && $drive->{file} =~ m/^(cdrom|none)$/;
1425 }
1426
1427 sub parse_hotplug_features {
1428     my ($data) = @_;
1429
1430     my $res = {};
1431
1432     return $res if $data eq '0';
1433
1434     $data = $confdesc->{hotplug}->{default} if $data eq '1';
1435
1436     foreach my $feature (PVE::Tools::split_list($data)) {
1437         if ($feature =~ m/^(network|disk|cpu|memory|usb)$/) {
1438             $res->{$1} = 1;
1439         } else {
1440             die "invalid hotplug feature '$feature'\n";
1441         }
1442     }
1443     return $res;
1444 }
1445
1446 PVE::JSONSchema::register_format('pve-hotplug-features', \&pve_verify_hotplug_features);
1447 sub pve_verify_hotplug_features {
1448     my ($value, $noerr) = @_;
1449
1450     return $value if parse_hotplug_features($value);
1451
1452     return undef if $noerr;
1453
1454     die "unable to parse hotplug option\n";
1455 }
1456
1457 # ideX = [volume=]volume-id[,media=d][,cyls=c,heads=h,secs=s[,trans=t]]
1458 #        [,snapshot=on|off][,cache=on|off][,format=f][,backup=yes|no]
1459 #        [,rerror=ignore|report|stop][,werror=enospc|ignore|report|stop]
1460 #        [,aio=native|threads][,discard=ignore|on][,detect_zeroes=on|off]
1461 #        [,iothread=on][,serial=serial][,model=model]
1462
1463 sub parse_drive {
1464     my ($key, $data) = @_;
1465
1466     my ($interface, $index);
1467
1468     if ($key =~ m/^([^\d]+)(\d+)$/) {
1469         $interface = $1;
1470         $index = $2;
1471     } else {
1472         return undef;
1473     }
1474
1475     my $desc = $key =~ /^unused\d+$/ ? $alldrive_fmt
1476                                      : $confdesc->{$key}->{format};
1477     if (!$desc) {
1478         warn "invalid drive key: $key\n";
1479         return undef;
1480     }
1481     my $res = eval { PVE::JSONSchema::parse_property_string($desc, $data) };
1482     return undef if !$res;
1483     $res->{interface} = $interface;
1484     $res->{index} = $index;
1485
1486     my $error = 0;
1487     foreach my $opt (qw(bps bps_rd bps_wr)) {
1488         if (my $bps = defined(delete $res->{$opt})) {
1489             if (defined($res->{"m$opt"})) {
1490                 warn "both $opt and m$opt specified\n";
1491                 ++$error;
1492                 next;
1493             }
1494             $res->{"m$opt"} = sprintf("%.3f", $bps / (1024*1024.0));
1495         }
1496     }
1497
1498     # can't use the schema's 'requires' because of the mbps* => bps* "transforming aliases"
1499     for my $requirement (
1500         [mbps_max => 'mbps'],
1501         [mbps_rd_max => 'mbps_rd'],
1502         [mbps_wr_max => 'mbps_wr'],
1503         [miops_max => 'miops'],
1504         [miops_rd_max => 'miops_rd'],
1505         [miops_wr_max => 'miops_wr'],
1506         [bps_max_length => 'mbps_max'],
1507         [bps_rd_max_length => 'mbps_rd_max'],
1508         [bps_wr_max_length => 'mbps_wr_max'],
1509         [iops_max_length => 'iops_max'],
1510         [iops_rd_max_length => 'iops_rd_max'],
1511         [iops_wr_max_length => 'iops_wr_max']) {
1512         my ($option, $requires) = @$requirement;
1513         if ($res->{$option} && !$res->{$requires}) {
1514             warn "$option requires $requires\n";
1515             ++$error;
1516         }
1517     }
1518
1519     return undef if $error;
1520
1521     return undef if $res->{mbps_rd} && $res->{mbps};
1522     return undef if $res->{mbps_wr} && $res->{mbps};
1523     return undef if $res->{iops_rd} && $res->{iops};
1524     return undef if $res->{iops_wr} && $res->{iops};
1525
1526     if ($res->{media} && ($res->{media} eq 'cdrom')) {
1527         return undef if $res->{snapshot} || $res->{trans} || $res->{format};
1528         return undef if $res->{heads} || $res->{secs} || $res->{cyls};
1529         return undef if $res->{interface} eq 'virtio';
1530     }
1531
1532     if (my $size = $res->{size}) {
1533         return undef if !defined($res->{size} = PVE::JSONSchema::parse_size($size));
1534     }
1535
1536     return $res;
1537 }
1538
1539 sub print_drive {
1540     my ($vmid, $drive) = @_;
1541     my $data = { %$drive };
1542     delete $data->{$_} for qw(index interface);
1543     return PVE::JSONSchema::print_property_string($data, $alldrive_fmt);
1544 }
1545
1546 sub scsi_inquiry {
1547     my($fh, $noerr) = @_;
1548
1549     my $SG_IO = 0x2285;
1550     my $SG_GET_VERSION_NUM = 0x2282;
1551
1552     my $versionbuf = "\x00" x 8;
1553     my $ret = ioctl($fh, $SG_GET_VERSION_NUM, $versionbuf);
1554     if (!$ret) {
1555         die "scsi ioctl SG_GET_VERSION_NUM failoed - $!\n" if !$noerr;
1556         return undef;
1557     }
1558     my $version = unpack("I", $versionbuf);
1559     if ($version < 30000) {
1560         die "scsi generic interface too old\n"  if !$noerr;
1561         return undef;
1562     }
1563
1564     my $buf = "\x00" x 36;
1565     my $sensebuf = "\x00" x 8;
1566     my $cmd = pack("C x3 C x1", 0x12, 36);
1567
1568     # see /usr/include/scsi/sg.h
1569     my $sg_io_hdr_t = "i i C C s I P P P I I i P C C C C S S i I I";
1570
1571     my $packet = pack($sg_io_hdr_t, ord('S'), -3, length($cmd),
1572                       length($sensebuf), 0, length($buf), $buf,
1573                       $cmd, $sensebuf, 6000);
1574
1575     $ret = ioctl($fh, $SG_IO, $packet);
1576     if (!$ret) {
1577         die "scsi ioctl SG_IO failed - $!\n" if !$noerr;
1578         return undef;
1579     }
1580
1581     my @res = unpack($sg_io_hdr_t, $packet);
1582     if ($res[17] || $res[18]) {
1583         die "scsi ioctl SG_IO status error - $!\n" if !$noerr;
1584         return undef;
1585     }
1586
1587     my $res = {};
1588     (my $byte0, my $byte1, $res->{vendor},
1589      $res->{product}, $res->{revision}) = unpack("C C x6 A8 A16 A4", $buf);
1590
1591     $res->{removable} = $byte1 & 128 ? 1 : 0;
1592     $res->{type} = $byte0 & 31;
1593
1594     return $res;
1595 }
1596
1597 sub path_is_scsi {
1598     my ($path) = @_;
1599
1600     my $fh = IO::File->new("+<$path") || return undef;
1601     my $res = scsi_inquiry($fh, 1);
1602     close($fh);
1603
1604     return $res;
1605 }
1606
1607 sub machine_type_is_q35 {
1608     my ($conf) = @_;
1609
1610     return $conf->{machine} && ($conf->{machine} =~ m/q35/) ? 1 : 0;
1611 }
1612
1613 sub print_tabletdevice_full {
1614     my ($conf) = @_;
1615
1616     my $q35 = machine_type_is_q35($conf);
1617
1618     # we use uhci for old VMs because tablet driver was buggy in older qemu
1619     my $usbbus = $q35 ? "ehci" : "uhci";
1620
1621     return "usb-tablet,id=tablet,bus=$usbbus.0,port=1";
1622 }
1623
1624 sub print_drivedevice_full {
1625     my ($storecfg, $conf, $vmid, $drive, $bridges) = @_;
1626
1627     my $device = '';
1628     my $maxdev = 0;
1629
1630     if ($drive->{interface} eq 'virtio') {
1631         my $pciaddr = print_pci_addr("$drive->{interface}$drive->{index}", $bridges);
1632         $device = "virtio-blk-pci,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}$pciaddr";
1633         $device .= ",iothread=iothread-$drive->{interface}$drive->{index}" if $drive->{iothread};
1634     } elsif ($drive->{interface} eq 'scsi') {
1635
1636         my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
1637         my $unit = $drive->{index} % $maxdev;
1638         my $devicetype = 'hd';
1639         my $path = '';
1640         if (drive_is_cdrom($drive)) {
1641             $devicetype = 'cd';
1642         } else {
1643             if ($drive->{file} =~ m|^/|) {
1644                 $path = $drive->{file};
1645                 if (my $info = path_is_scsi($path)) {
1646                     if ($info->{type} == 0 && $drive->{scsiblock}) {
1647                         $devicetype = 'block';
1648                     } elsif ($info->{type} == 1) { # tape
1649                         $devicetype = 'generic';
1650                     }
1651                 }
1652             } else {
1653                  $path = PVE::Storage::path($storecfg, $drive->{file});
1654             }
1655
1656             if($path =~ m/^iscsi\:\/\//){
1657                 $devicetype = 'generic';
1658             }
1659         }
1660
1661         if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)){
1662             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,scsi-id=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1663         } else {
1664             $device = "scsi-$devicetype,bus=$controller_prefix$controller.0,channel=0,scsi-id=0,lun=$drive->{index},drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1665         }
1666
1667     } elsif ($drive->{interface} eq 'ide'){
1668         $maxdev = 2;
1669         my $controller = int($drive->{index} / $maxdev);
1670         my $unit = $drive->{index} % $maxdev;
1671         my $devicetype = ($drive->{media} && $drive->{media} eq 'cdrom') ? "cd" : "hd";
1672
1673         $device = "ide-$devicetype,bus=ide.$controller,unit=$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1674         if ($devicetype eq 'hd' && (my $model = $drive->{model})) {
1675             $model = URI::Escape::uri_unescape($model);
1676             $device .= ",model=$model";
1677         }
1678     } elsif ($drive->{interface} eq 'sata'){
1679         my $controller = int($drive->{index} / $MAX_SATA_DISKS);
1680         my $unit = $drive->{index} % $MAX_SATA_DISKS;
1681         $device = "ide-drive,bus=ahci$controller.$unit,drive=drive-$drive->{interface}$drive->{index},id=$drive->{interface}$drive->{index}";
1682     } elsif ($drive->{interface} eq 'usb') {
1683         die "implement me";
1684         #  -device ide-drive,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
1685     } else {
1686         die "unsupported interface type";
1687     }
1688
1689     $device .= ",bootindex=$drive->{bootindex}" if $drive->{bootindex};
1690
1691     return $device;
1692 }
1693
1694 sub get_initiator_name {
1695     my $initiator;
1696
1697     my $fh = IO::File->new('/etc/iscsi/initiatorname.iscsi') || return undef;
1698     while (defined(my $line = <$fh>)) {
1699         next if $line !~ m/^\s*InitiatorName\s*=\s*([\.\-:\w]+)/;
1700         $initiator = $1;
1701         last;
1702     }
1703     $fh->close();
1704
1705     return $initiator;
1706 }
1707
1708 sub print_drive_full {
1709     my ($storecfg, $vmid, $drive) = @_;
1710
1711     my $path;
1712     my $volid = $drive->{file};
1713     my $format;
1714
1715     if (drive_is_cdrom($drive)) {
1716         $path = get_iso_path($storecfg, $vmid, $volid);
1717     } else {
1718         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
1719         if ($storeid) {
1720             $path = PVE::Storage::path($storecfg, $volid);
1721             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
1722             $format = qemu_img_format($scfg, $volname);
1723         } else {
1724             $path = $volid;
1725             $format = "raw";
1726         }
1727    }
1728
1729     my $opts = '';
1730     my @qemu_drive_options = qw(heads secs cyls trans media format cache rerror werror aio discard);
1731     foreach my $o (@qemu_drive_options) {
1732         $opts .= ",$o=$drive->{$o}" if defined($drive->{$o});
1733     }
1734
1735     # snapshot only accepts on|off
1736     if (defined($drive->{snapshot})) {
1737         my $v = $drive->{snapshot} ? 'on' : 'off';
1738         $opts .= ",snapshot=$v";
1739     }
1740
1741     foreach my $type (['', '-total'], [_rd => '-read'], [_wr => '-write']) {
1742         my ($dir, $qmpname) = @$type;
1743         if (my $v = $drive->{"mbps$dir"}) {
1744             $opts .= ",throttling.bps$qmpname=".int($v*1024*1024);
1745         }
1746         if (my $v = $drive->{"mbps${dir}_max"}) {
1747             $opts .= ",throttling.bps$qmpname-max=".int($v*1024*1024);
1748         }
1749         if (my $v = $drive->{"bps${dir}_max_length"}) {
1750             $opts .= ",throttling.bps$qmpname-max-length=$v";
1751         }
1752         if (my $v = $drive->{"iops${dir}"}) {
1753             $opts .= ",throttling.iops$qmpname=$v";
1754         }
1755         if (my $v = $drive->{"iops${dir}_max"}) {
1756             $opts .= ",throttling.iops$qmpname-max=$v";
1757         }
1758         if (my $v = $drive->{"iops${dir}_max_length"}) {
1759             $opts .= ",throttling.iops$qmpname-max-length=$v";
1760         }
1761     }
1762
1763     if (my $serial = $drive->{serial}) {
1764         $serial = URI::Escape::uri_unescape($serial);
1765         $opts .= ",serial=$serial";
1766     }
1767
1768     $opts .= ",format=$format" if $format && !$drive->{format};
1769
1770     my $cache_direct = 0;
1771
1772     if (my $cache = $drive->{cache}) {
1773         $cache_direct = $cache =~ /^(?:off|none|directsync)$/;
1774     } elsif (!drive_is_cdrom($drive)) {
1775         $opts .= ",cache=none";
1776         $cache_direct = 1;
1777     }
1778
1779     # aio native works only with O_DIRECT
1780     if (!$drive->{aio}) {
1781         if($cache_direct) {
1782             $opts .= ",aio=native";
1783         } else {
1784             $opts .= ",aio=threads";
1785         }
1786     }
1787
1788     if (!drive_is_cdrom($drive)) {
1789         my $detectzeroes;
1790         if (defined($drive->{detect_zeroes}) && !$drive->{detect_zeroes}) {
1791             $detectzeroes = 'off';
1792         } elsif ($drive->{discard}) {
1793             $detectzeroes = $drive->{discard} eq 'on' ? 'unmap' : 'on';
1794         } else {
1795             # This used to be our default with discard not being specified:
1796             $detectzeroes = 'on';
1797         }
1798         $opts .= ",detect-zeroes=$detectzeroes" if $detectzeroes;
1799     }
1800
1801     my $pathinfo = $path ? "file=$path," : '';
1802
1803     return "${pathinfo}if=none,id=drive-$drive->{interface}$drive->{index}$opts";
1804 }
1805
1806 sub print_netdevice_full {
1807     my ($vmid, $conf, $net, $netid, $bridges, $use_old_bios_files) = @_;
1808
1809     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
1810
1811     my $device = $net->{model};
1812     if ($net->{model} eq 'virtio') {
1813          $device = 'virtio-net-pci';
1814      };
1815
1816     my $pciaddr = print_pci_addr("$netid", $bridges);
1817     my $tmpstr = "$device,mac=$net->{macaddr},netdev=$netid$pciaddr,id=$netid";
1818     if ($net->{queues} && $net->{queues} > 1 && $net->{model} eq 'virtio'){
1819         #Consider we have N queues, the number of vectors needed is 2*N + 2 (plus one config interrupt and control vq)
1820         my $vectors = $net->{queues} * 2 + 2;
1821         $tmpstr .= ",vectors=$vectors,mq=on";
1822     }
1823     $tmpstr .= ",bootindex=$net->{bootindex}" if $net->{bootindex} ;
1824
1825     if ($use_old_bios_files) {
1826         my $romfile;
1827         if ($device eq 'virtio-net-pci') {
1828             $romfile = 'pxe-virtio.rom';
1829         } elsif ($device eq 'e1000') {
1830             $romfile = 'pxe-e1000.rom';
1831         } elsif ($device eq 'ne2k') {
1832             $romfile = 'pxe-ne2k_pci.rom';
1833         } elsif ($device eq 'pcnet') {
1834             $romfile = 'pxe-pcnet.rom';
1835         } elsif ($device eq 'rtl8139') {
1836             $romfile = 'pxe-rtl8139.rom';
1837         }
1838         $tmpstr .= ",romfile=$romfile" if $romfile;
1839     }
1840
1841     return $tmpstr;
1842 }
1843
1844 sub print_netdev_full {
1845     my ($vmid, $conf, $net, $netid, $hotplug) = @_;
1846
1847     my $i = '';
1848     if ($netid =~ m/^net(\d+)$/) {
1849         $i = int($1);
1850     }
1851
1852     die "got strange net id '$i'\n" if $i >= ${MAX_NETS};
1853
1854     my $ifname = "tap${vmid}i$i";
1855
1856     # kvm uses TUNSETIFF ioctl, and that limits ifname length
1857     die "interface name '$ifname' is too long (max 15 character)\n"
1858         if length($ifname) >= 16;
1859
1860     my $vhostparam = '';
1861     $vhostparam = ',vhost=on' if $kernel_has_vhost_net && $net->{model} eq 'virtio';
1862
1863     my $vmname = $conf->{name} || "vm$vmid";
1864
1865     my $netdev = "";
1866     my $script = $hotplug ? "pve-bridge-hotplug" : "pve-bridge";
1867
1868     if ($net->{bridge}) {
1869         $netdev = "type=tap,id=$netid,ifname=${ifname},script=/var/lib/qemu-server/$script,downscript=/var/lib/qemu-server/pve-bridgedown$vhostparam";
1870     } else {
1871         $netdev = "type=user,id=$netid,hostname=$vmname";
1872     }
1873
1874     $netdev .= ",queues=$net->{queues}" if ($net->{queues} && $net->{model} eq 'virtio');
1875
1876     return $netdev;
1877 }
1878
1879
1880 sub print_cpu_device {
1881     my ($conf, $id) = @_;
1882
1883     my $kvm = $conf->{kvm} // 1;
1884     my $cpu = $kvm ? "kvm64" : "qemu64";
1885     if (my $cputype = $conf->{cpu}) {
1886         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
1887             or die "Cannot parse cpu description: $cputype\n";
1888         $cpu = $cpuconf->{cputype};
1889     }
1890
1891     my $cores = $conf->{cores} || 1;
1892
1893     my $current_core = ($id - 1) % $cores;
1894     my $current_socket = int(($id - 1 - $current_core)/$cores);
1895
1896     return "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
1897 }
1898
1899 sub drive_is_cloudinit {
1900     my ($drive) = @_;
1901     return $drive->{file} =~ m@[:/]vm-\d+-cloudinit(?:\.$QEMU_FORMAT_RE)?$@;
1902 }
1903
1904 sub drive_is_cdrom {
1905     my ($drive, $exclude_cloudinit) = @_;
1906
1907     return 0 if $exclude_cloudinit && drive_is_cloudinit($drive);
1908
1909     return $drive && $drive->{media} && ($drive->{media} eq 'cdrom');
1910
1911 }
1912
1913 sub parse_number_sets {
1914     my ($set) = @_;
1915     my $res = [];
1916     foreach my $part (split(/;/, $set)) {
1917         if ($part =~ /^\s*(\d+)(?:-(\d+))?\s*$/) {
1918             die "invalid range: $part ($2 < $1)\n" if defined($2) && $2 < $1;
1919             push @$res, [ $1, $2 ];
1920         } else {
1921             die "invalid range: $part\n";
1922         }
1923     }
1924     return $res;
1925 }
1926
1927 sub parse_numa {
1928     my ($data) = @_;
1929
1930     my $res = PVE::JSONSchema::parse_property_string($numa_fmt, $data);
1931     $res->{cpus} = parse_number_sets($res->{cpus}) if defined($res->{cpus});
1932     $res->{hostnodes} = parse_number_sets($res->{hostnodes}) if defined($res->{hostnodes});
1933     return $res;
1934 }
1935
1936 sub parse_hostpci {
1937     my ($value) = @_;
1938
1939     return undef if !$value;
1940
1941     my $res = PVE::JSONSchema::parse_property_string($hostpci_fmt, $value);
1942
1943     my @idlist = split(/;/, $res->{host});
1944     delete $res->{host};
1945     foreach my $id (@idlist) {
1946         if ($id =~ /^$PCIRE$/) {
1947             if (defined($2)) {
1948                 push @{$res->{pciid}}, { id => $1, function => $2 };
1949             } else {
1950                 my $pcidevices = lspci($1);
1951                 $res->{pciid} = $pcidevices->{$1};
1952             }
1953         } else {
1954             # should have been caught by parse_property_string already
1955             die "failed to parse PCI id: $id\n";
1956         }
1957     }
1958     return $res;
1959 }
1960
1961 # netX: e1000=XX:XX:XX:XX:XX:XX,bridge=vmbr0,rate=<mbps>
1962 sub parse_net {
1963     my ($data) = @_;
1964
1965     my $res = eval { PVE::JSONSchema::parse_property_string($net_fmt, $data) };
1966     if ($@) {
1967         warn $@;
1968         return undef;
1969     }
1970     if (!defined($res->{macaddr})) {
1971         my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
1972         $res->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix});
1973     }
1974     $res->{macaddr} = PVE::Tools::random_ether_addr() if !defined($res->{macaddr});
1975     return $res;
1976 }
1977
1978 # ipconfigX ip=cidr,gw=ip,ip6=cidr,gw6=ip
1979 sub parse_ipconfig {
1980     my ($data) = @_;
1981
1982     my $res = eval { PVE::JSONSchema::parse_property_string($ipconfig_fmt, $data) };
1983     if ($@) {
1984         warn $@;
1985         return undef;
1986     }
1987
1988     if ($res->{gw} && !$res->{ip}) {
1989         warn 'gateway specified without specifying an IP address';
1990         return undef;
1991     }
1992     if ($res->{gw6} && !$res->{ip6}) {
1993         warn 'IPv6 gateway specified without specifying an IPv6 address';
1994         return undef;
1995     }
1996     if ($res->{gw} && $res->{ip} eq 'dhcp') {
1997         warn 'gateway specified together with DHCP';
1998         return undef;
1999     }
2000     if ($res->{gw6} && $res->{ip6} !~ /^$IPV6RE/) {
2001         # gw6 + auto/dhcp
2002         warn "IPv6 gateway specified together with $res->{ip6} address";
2003         return undef;
2004     }
2005
2006     if (!$res->{ip} && !$res->{ip6}) {
2007         return { ip => 'dhcp', ip6 => 'dhcp' };
2008     }
2009
2010     return $res;
2011 }
2012
2013 sub print_net {
2014     my $net = shift;
2015
2016     return PVE::JSONSchema::print_property_string($net, $net_fmt);
2017 }
2018
2019 sub add_random_macs {
2020     my ($settings) = @_;
2021
2022     foreach my $opt (keys %$settings) {
2023         next if $opt !~ m/^net(\d+)$/;
2024         my $net = parse_net($settings->{$opt});
2025         next if !$net;
2026         $settings->{$opt} = print_net($net);
2027     }
2028 }
2029
2030 sub vm_is_volid_owner {
2031     my ($storecfg, $vmid, $volid) = @_;
2032
2033     if ($volid !~  m|^/|) {
2034         my ($path, $owner);
2035         eval { ($path, $owner) = PVE::Storage::path($storecfg, $volid); };
2036         if ($owner && ($owner == $vmid)) {
2037             return 1;
2038         }
2039     }
2040
2041     return undef;
2042 }
2043
2044 sub split_flagged_list {
2045     my $text = shift || '';
2046     $text =~ s/[,;]/ /g;
2047     $text =~ s/^\s+//;
2048     return { map { /^(!?)(.*)$/ && ($2, $1) } ($text =~ /\S+/g) };
2049 }
2050
2051 sub join_flagged_list {
2052     my ($how, $lst) = @_;
2053     join $how, map { $lst->{$_} . $_ } keys %$lst;
2054 }
2055
2056 sub vmconfig_delete_pending_option {
2057     my ($conf, $key, $force) = @_;
2058
2059     delete $conf->{pending}->{$key};
2060     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2061     $pending_delete_hash->{$key} = $force ? '!' : '';
2062     $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2063 }
2064
2065 sub vmconfig_undelete_pending_option {
2066     my ($conf, $key) = @_;
2067
2068     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
2069     delete $pending_delete_hash->{$key};
2070
2071     if (%$pending_delete_hash) {
2072         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2073     } else {
2074         delete $conf->{pending}->{delete};
2075     }
2076 }
2077
2078 sub vmconfig_register_unused_drive {
2079     my ($storecfg, $vmid, $conf, $drive) = @_;
2080
2081     if (drive_is_cloudinit($drive)) {
2082         eval { PVE::Storage::vdisk_free($storecfg, $drive->{file}) };
2083         warn $@ if $@;
2084     } elsif (!drive_is_cdrom($drive)) {
2085         my $volid = $drive->{file};
2086         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
2087             PVE::QemuConfig->add_unused_volume($conf, $volid, $vmid);
2088         }
2089     }
2090 }
2091
2092 sub vmconfig_cleanup_pending {
2093     my ($conf) = @_;
2094
2095     # remove pending changes when nothing changed
2096     my $changes;
2097     foreach my $opt (keys %{$conf->{pending}}) {
2098         if (defined($conf->{$opt}) && ($conf->{pending}->{$opt} eq  $conf->{$opt})) {
2099             $changes = 1;
2100             delete $conf->{pending}->{$opt};
2101         }
2102     }
2103
2104     my $current_delete_hash = split_flagged_list($conf->{pending}->{delete});
2105     my $pending_delete_hash = {};
2106     while (my ($opt, $force) = each %$current_delete_hash) {
2107         if (defined($conf->{$opt})) {
2108             $pending_delete_hash->{$opt} = $force;
2109         } else {
2110             $changes = 1;
2111         }
2112     }
2113
2114     if (%$pending_delete_hash) {
2115         $conf->{pending}->{delete} = join_flagged_list(',', $pending_delete_hash);
2116     } else {
2117         delete $conf->{pending}->{delete};
2118     }
2119
2120     return $changes;
2121 }
2122
2123 # smbios: [manufacturer=str][,product=str][,version=str][,serial=str][,uuid=uuid][,sku=str][,family=str]
2124 my $smbios1_fmt = {
2125     uuid => {
2126         type => 'string',
2127         pattern => '[a-fA-F0-9]{8}(?:-[a-fA-F0-9]{4}){3}-[a-fA-F0-9]{12}',
2128         format_description => 'UUID',
2129         description => "Set SMBIOS1 UUID.",
2130         optional => 1,
2131     },
2132     version => {
2133         type => 'string',
2134         pattern => '\S+',
2135         format_description => 'string',
2136         description => "Set SMBIOS1 version.",
2137         optional => 1,
2138     },
2139     serial => {
2140         type => 'string',
2141         pattern => '\S+',
2142         format_description => 'string',
2143         description => "Set SMBIOS1 serial number.",
2144         optional => 1,
2145     },
2146     manufacturer => {
2147         type => 'string',
2148         pattern => '\S+',
2149         format_description => 'string',
2150         description => "Set SMBIOS1 manufacturer.",
2151         optional => 1,
2152     },
2153     product => {
2154         type => 'string',
2155         pattern => '\S+',
2156         format_description => 'string',
2157         description => "Set SMBIOS1 product ID.",
2158         optional => 1,
2159     },
2160     sku => {
2161         type => 'string',
2162         pattern => '\S+',
2163         format_description => 'string',
2164         description => "Set SMBIOS1 SKU string.",
2165         optional => 1,
2166     },
2167     family => {
2168         type => 'string',
2169         pattern => '\S+',
2170         format_description => 'string',
2171         description => "Set SMBIOS1 family string.",
2172         optional => 1,
2173     },
2174 };
2175
2176 sub parse_smbios1 {
2177     my ($data) = @_;
2178
2179     my $res = eval { PVE::JSONSchema::parse_property_string($smbios1_fmt, $data) };
2180     warn $@ if $@;
2181     return $res;
2182 }
2183
2184 sub print_smbios1 {
2185     my ($smbios1) = @_;
2186     return PVE::JSONSchema::print_property_string($smbios1, $smbios1_fmt);
2187 }
2188
2189 PVE::JSONSchema::register_format('pve-qm-smbios1', $smbios1_fmt);
2190
2191 PVE::JSONSchema::register_format('pve-qm-bootdisk', \&verify_bootdisk);
2192 sub verify_bootdisk {
2193     my ($value, $noerr) = @_;
2194
2195     return $value if is_valid_drivename($value);
2196
2197     return undef if $noerr;
2198
2199     die "invalid boot disk '$value'\n";
2200 }
2201
2202 sub parse_watchdog {
2203     my ($value) = @_;
2204
2205     return undef if !$value;
2206
2207     my $res = eval { PVE::JSONSchema::parse_property_string($watchdog_fmt, $value) };
2208     warn $@ if $@;
2209     return $res;
2210 }
2211
2212 PVE::JSONSchema::register_format('pve-qm-usb-device', \&verify_usb_device);
2213 sub verify_usb_device {
2214     my ($value, $noerr) = @_;
2215
2216     return $value if parse_usb_device($value);
2217
2218     return undef if $noerr;
2219
2220     die "unable to parse usb device\n";
2221 }
2222
2223 # add JSON properties for create and set function
2224 sub json_config_properties {
2225     my $prop = shift;
2226
2227     foreach my $opt (keys %$confdesc) {
2228         next if $opt eq 'parent' || $opt eq 'snaptime' || $opt eq 'vmstate';
2229         $prop->{$opt} = $confdesc->{$opt};
2230     }
2231
2232     return $prop;
2233 }
2234
2235 # return copy of $confdesc_cloudinit to generate documentation
2236 sub cloudinit_config_properties {
2237
2238     return dclone($confdesc_cloudinit);
2239 }
2240
2241 sub check_type {
2242     my ($key, $value) = @_;
2243
2244     die "unknown setting '$key'\n" if !$confdesc->{$key};
2245
2246     my $type = $confdesc->{$key}->{type};
2247
2248     if (!defined($value)) {
2249         die "got undefined value\n";
2250     }
2251
2252     if ($value =~ m/[\n\r]/) {
2253         die "property contains a line feed\n";
2254     }
2255
2256     if ($type eq 'boolean') {
2257         return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i);
2258         return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i);
2259         die "type check ('boolean') failed - got '$value'\n";
2260     } elsif ($type eq 'integer') {
2261         return int($1) if $value =~ m/^(\d+)$/;
2262         die "type check ('integer') failed - got '$value'\n";
2263     } elsif ($type eq 'number') {
2264         return $value if $value =~ m/^(\d+)(\.\d+)?$/;
2265         die "type check ('number') failed - got '$value'\n";
2266     } elsif ($type eq 'string') {
2267         if (my $fmt = $confdesc->{$key}->{format}) {
2268             PVE::JSONSchema::check_format($fmt, $value);
2269             return $value;
2270         }
2271         $value =~ s/^\"(.*)\"$/$1/;
2272         return $value;
2273     } else {
2274         die "internal error"
2275     }
2276 }
2277
2278 sub check_iommu_support{
2279     #fixme : need to check IOMMU support
2280     #http://www.linux-kvm.org/page/How_to_assign_devices_with_VT-d_in_KVM
2281
2282     my $iommu=1;
2283     return $iommu;
2284
2285 }
2286
2287 sub touch_config {
2288     my ($vmid) = @_;
2289
2290     my $conf = PVE::QemuConfig->config_file($vmid);
2291     utime undef, undef, $conf;
2292 }
2293
2294 sub destroy_vm {
2295     my ($storecfg, $vmid, $keep_empty_config, $skiplock) = @_;
2296
2297     my $conffile = PVE::QemuConfig->config_file($vmid);
2298
2299     my $conf = PVE::QemuConfig->load_config($vmid);
2300
2301     PVE::QemuConfig->check_lock($conf) if !$skiplock;
2302
2303     if ($conf->{template}) {
2304         # check if any base image is still used by a linked clone
2305         foreach_drive($conf, sub {
2306                 my ($ds, $drive) = @_;
2307
2308                 return if drive_is_cdrom($drive);
2309
2310                 my $volid = $drive->{file};
2311
2312                 return if !$volid || $volid =~ m|^/|;
2313
2314                 die "base volume '$volid' is still in use by linked cloned\n"
2315                     if PVE::Storage::volume_is_base_and_used($storecfg, $volid);
2316
2317         });
2318     }
2319
2320     # only remove disks owned by this VM
2321     foreach_drive($conf, sub {
2322         my ($ds, $drive) = @_;
2323
2324         return if drive_is_cdrom($drive, 1);
2325
2326         my $volid = $drive->{file};
2327
2328         return if !$volid || $volid =~ m|^/|;
2329
2330         my ($path, $owner) = PVE::Storage::path($storecfg, $volid);
2331         return if !$path || !$owner || ($owner != $vmid);
2332
2333         eval {
2334             PVE::Storage::vdisk_free($storecfg, $volid);
2335         };
2336         warn "Could not remove disk '$volid', check manually: $@" if $@;
2337
2338     });
2339
2340     if ($keep_empty_config) {
2341         PVE::Tools::file_set_contents($conffile, "memory: 128\n");
2342     } else {
2343         unlink $conffile;
2344     }
2345
2346     # also remove unused disk
2347     eval {
2348         my $dl = PVE::Storage::vdisk_list($storecfg, undef, $vmid);
2349
2350         eval {
2351             PVE::Storage::foreach_volid($dl, sub {
2352                 my ($volid, $sid, $volname, $d) = @_;
2353                 PVE::Storage::vdisk_free($storecfg, $volid);
2354             });
2355         };
2356         warn $@ if $@;
2357
2358     };
2359     warn $@ if $@;
2360 }
2361
2362 sub parse_vm_config {
2363     my ($filename, $raw) = @_;
2364
2365     return undef if !defined($raw);
2366
2367     my $res = {
2368         digest => Digest::SHA::sha1_hex($raw),
2369         snapshots => {},
2370         pending => {},
2371     };
2372
2373     $filename =~ m|/qemu-server/(\d+)\.conf$|
2374         || die "got strange filename '$filename'";
2375
2376     my $vmid = $1;
2377
2378     my $conf = $res;
2379     my $descr;
2380     my $section = '';
2381
2382     my @lines = split(/\n/, $raw);
2383     foreach my $line (@lines) {
2384         next if $line =~ m/^\s*$/;
2385
2386         if ($line =~ m/^\[PENDING\]\s*$/i) {
2387             $section = 'pending';
2388             if (defined($descr)) {
2389                 $descr =~ s/\s+$//;
2390                 $conf->{description} = $descr;
2391             }
2392             $descr = undef;
2393             $conf = $res->{$section} = {};
2394             next;
2395
2396         } elsif ($line =~ m/^\[([a-z][a-z0-9_\-]+)\]\s*$/i) {
2397             $section = $1;
2398             if (defined($descr)) {
2399                 $descr =~ s/\s+$//;
2400                 $conf->{description} = $descr;
2401             }
2402             $descr = undef;
2403             $conf = $res->{snapshots}->{$section} = {};
2404             next;
2405         }
2406
2407         if ($line =~ m/^\#(.*)\s*$/) {
2408             $descr = '' if !defined($descr);
2409             $descr .= PVE::Tools::decode_text($1) . "\n";
2410             next;
2411         }
2412
2413         if ($line =~ m/^(description):\s*(.*\S)\s*$/) {
2414             $descr = '' if !defined($descr);
2415             $descr .= PVE::Tools::decode_text($2);
2416         } elsif ($line =~ m/snapstate:\s*(prepare|delete)\s*$/) {
2417             $conf->{snapstate} = $1;
2418         } elsif ($line =~ m/^(args):\s*(.*\S)\s*$/) {
2419             my $key = $1;
2420             my $value = $2;
2421             $conf->{$key} = $value;
2422         } elsif ($line =~ m/^delete:\s*(.*\S)\s*$/) {
2423             my $value = $1;
2424             if ($section eq 'pending') {
2425                 $conf->{delete} = $value; # we parse this later
2426             } else {
2427                 warn "vm $vmid - propertry 'delete' is only allowed in [PENDING]\n";
2428             }
2429         } elsif ($line =~ m/^([a-z][a-z_]*\d*):\s*(.+?)\s*$/) {
2430             my $key = $1;
2431             my $value = $2;
2432             eval { $value = check_type($key, $value); };
2433             if ($@) {
2434                 warn "vm $vmid - unable to parse value of '$key' - $@";
2435             } else {
2436                 $key = 'ide2' if $key eq 'cdrom';
2437                 my $fmt = $confdesc->{$key}->{format};
2438                 if ($fmt && $fmt =~ /^pve-qm-(?:ide|scsi|virtio|sata)$/) {
2439                     my $v = parse_drive($key, $value);
2440                     if (my $volid = filename_to_volume_id($vmid, $v->{file}, $v->{media})) {
2441                         $v->{file} = $volid;
2442                         $value = print_drive($vmid, $v);
2443                     } else {
2444                         warn "vm $vmid - unable to parse value of '$key'\n";
2445                         next;
2446                     }
2447                 }
2448
2449                 $conf->{$key} = $value;
2450             }
2451         }
2452     }
2453
2454     if (defined($descr)) {
2455         $descr =~ s/\s+$//;
2456         $conf->{description} = $descr;
2457     }
2458     delete $res->{snapstate}; # just to be sure
2459
2460     return $res;
2461 }
2462
2463 sub write_vm_config {
2464     my ($filename, $conf) = @_;
2465
2466     delete $conf->{snapstate}; # just to be sure
2467
2468     if ($conf->{cdrom}) {
2469         die "option ide2 conflicts with cdrom\n" if $conf->{ide2};
2470         $conf->{ide2} = $conf->{cdrom};
2471         delete $conf->{cdrom};
2472     }
2473
2474     # we do not use 'smp' any longer
2475     if ($conf->{sockets}) {
2476         delete $conf->{smp};
2477     } elsif ($conf->{smp}) {
2478         $conf->{sockets} = $conf->{smp};
2479         delete $conf->{cores};
2480         delete $conf->{smp};
2481     }
2482
2483     my $used_volids = {};
2484
2485     my $cleanup_config = sub {
2486         my ($cref, $pending, $snapname) = @_;
2487
2488         foreach my $key (keys %$cref) {
2489             next if $key eq 'digest' || $key eq 'description' || $key eq 'snapshots' ||
2490                 $key eq 'snapstate' || $key eq 'pending';
2491             my $value = $cref->{$key};
2492             if ($key eq 'delete') {
2493                 die "propertry 'delete' is only allowed in [PENDING]\n"
2494                     if !$pending;
2495                 # fixme: check syntax?
2496                 next;
2497             }
2498             eval { $value = check_type($key, $value); };
2499             die "unable to parse value of '$key' - $@" if $@;
2500
2501             $cref->{$key} = $value;
2502
2503             if (!$snapname && is_valid_drivename($key)) {
2504                 my $drive = parse_drive($key, $value);
2505                 $used_volids->{$drive->{file}} = 1 if $drive && $drive->{file};
2506             }
2507         }
2508     };
2509
2510     &$cleanup_config($conf);
2511
2512     &$cleanup_config($conf->{pending}, 1);
2513
2514     foreach my $snapname (keys %{$conf->{snapshots}}) {
2515         die "internal error" if $snapname eq 'pending';
2516         &$cleanup_config($conf->{snapshots}->{$snapname}, undef, $snapname);
2517     }
2518
2519     # remove 'unusedX' settings if we re-add a volume
2520     foreach my $key (keys %$conf) {
2521         my $value = $conf->{$key};
2522         if ($key =~ m/^unused/ && $used_volids->{$value}) {
2523             delete $conf->{$key};
2524         }
2525     }
2526
2527     my $generate_raw_config = sub {
2528         my ($conf, $pending) = @_;
2529
2530         my $raw = '';
2531
2532         # add description as comment to top of file
2533         if (defined(my $descr = $conf->{description})) {
2534             if ($descr) {
2535                 foreach my $cl (split(/\n/, $descr)) {
2536                     $raw .= '#' .  PVE::Tools::encode_text($cl) . "\n";
2537                 }
2538             } else {
2539                 $raw .= "#\n" if $pending;
2540             }
2541         }
2542
2543         foreach my $key (sort keys %$conf) {
2544             next if $key eq 'digest' || $key eq 'description' || $key eq 'pending' || $key eq 'snapshots';
2545             $raw .= "$key: $conf->{$key}\n";
2546         }
2547         return $raw;
2548     };
2549
2550     my $raw = &$generate_raw_config($conf);
2551
2552     if (scalar(keys %{$conf->{pending}})){
2553         $raw .= "\n[PENDING]\n";
2554         $raw .= &$generate_raw_config($conf->{pending}, 1);
2555     }
2556
2557     foreach my $snapname (sort keys %{$conf->{snapshots}}) {
2558         $raw .= "\n[$snapname]\n";
2559         $raw .= &$generate_raw_config($conf->{snapshots}->{$snapname});
2560     }
2561
2562     return $raw;
2563 }
2564
2565 sub load_defaults {
2566
2567     my $res = {};
2568
2569     # we use static defaults from our JSON schema configuration
2570     foreach my $key (keys %$confdesc) {
2571         if (defined(my $default = $confdesc->{$key}->{default})) {
2572             $res->{$key} = $default;
2573         }
2574     }
2575
2576     my $conf = PVE::Cluster::cfs_read_file('datacenter.cfg');
2577     $res->{keyboard} = $conf->{keyboard} if $conf->{keyboard};
2578
2579     return $res;
2580 }
2581
2582 sub config_list {
2583     my $vmlist = PVE::Cluster::get_vmlist();
2584     my $res = {};
2585     return $res if !$vmlist || !$vmlist->{ids};
2586     my $ids = $vmlist->{ids};
2587
2588     foreach my $vmid (keys %$ids) {
2589         my $d = $ids->{$vmid};
2590         next if !$d->{node} || $d->{node} ne $nodename;
2591         next if !$d->{type} || $d->{type} ne 'qemu';
2592         $res->{$vmid}->{exists} = 1;
2593     }
2594     return $res;
2595 }
2596
2597 # test if VM uses local resources (to prevent migration)
2598 sub check_local_resources {
2599     my ($conf, $noerr) = @_;
2600
2601     my $loc_res = 0;
2602
2603     $loc_res = 1 if $conf->{hostusb}; # old syntax
2604     $loc_res = 1 if $conf->{hostpci}; # old syntax
2605
2606     foreach my $k (keys %$conf) {
2607         next if $k =~ m/^usb/ && ($conf->{$k} eq 'spice');
2608         # sockets are safe: they will recreated be on the target side post-migrate
2609         next if $k =~ m/^serial/ && ($conf->{$k} eq 'socket');
2610         $loc_res = 1 if $k =~ m/^(usb|hostpci|serial|parallel)\d+$/;
2611     }
2612
2613     die "VM uses local resources\n" if $loc_res && !$noerr;
2614
2615     return $loc_res;
2616 }
2617
2618 # check if used storages are available on all nodes (use by migrate)
2619 sub check_storage_availability {
2620     my ($storecfg, $conf, $node) = @_;
2621
2622     foreach_drive($conf, sub {
2623         my ($ds, $drive) = @_;
2624
2625         my $volid = $drive->{file};
2626         return if !$volid;
2627
2628         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2629         return if !$sid;
2630
2631         # check if storage is available on both nodes
2632         my $scfg = PVE::Storage::storage_check_node($storecfg, $sid);
2633         PVE::Storage::storage_check_node($storecfg, $sid, $node);
2634    });
2635 }
2636
2637 # list nodes where all VM images are available (used by has_feature API)
2638 sub shared_nodes {
2639     my ($conf, $storecfg) = @_;
2640
2641     my $nodelist = PVE::Cluster::get_nodelist();
2642     my $nodehash = { map { $_ => 1 } @$nodelist };
2643     my $nodename = PVE::INotify::nodename();
2644
2645     foreach_drive($conf, sub {
2646         my ($ds, $drive) = @_;
2647
2648         my $volid = $drive->{file};
2649         return if !$volid;
2650
2651         my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
2652         if ($storeid) {
2653             my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
2654             if ($scfg->{disable}) {
2655                 $nodehash = {};
2656             } elsif (my $avail = $scfg->{nodes}) {
2657                 foreach my $node (keys %$nodehash) {
2658                     delete $nodehash->{$node} if !$avail->{$node};
2659                 }
2660             } elsif (!$scfg->{shared}) {
2661                 foreach my $node (keys %$nodehash) {
2662                     delete $nodehash->{$node} if $node ne $nodename
2663                 }
2664             }
2665         }
2666     });
2667
2668     return $nodehash
2669 }
2670
2671 sub check_cmdline {
2672     my ($pidfile, $pid) = @_;
2673
2674     my $fh = IO::File->new("/proc/$pid/cmdline", "r");
2675     if (defined($fh)) {
2676         my $line = <$fh>;
2677         $fh->close;
2678         return undef if !$line;
2679         my @param = split(/\0/, $line);
2680
2681         my $cmd = $param[0];
2682         return if !$cmd || ($cmd !~ m|kvm$| && $cmd !~ m|qemu-system-x86_64$|);
2683
2684         for (my $i = 0; $i < scalar (@param); $i++) {
2685             my $p = $param[$i];
2686             next if !$p;
2687             if (($p eq '-pidfile') || ($p eq '--pidfile')) {
2688                 my $p = $param[$i+1];
2689                 return 1 if $p && ($p eq $pidfile);
2690                 return undef;
2691             }
2692         }
2693     }
2694     return undef;
2695 }
2696
2697 sub check_running {
2698     my ($vmid, $nocheck, $node) = @_;
2699
2700     my $filename = PVE::QemuConfig->config_file($vmid, $node);
2701
2702     die "unable to find configuration file for VM $vmid - no such machine\n"
2703         if !$nocheck && ! -f $filename;
2704
2705     my $pidfile = pidfile_name($vmid);
2706
2707     if (my $fd = IO::File->new("<$pidfile")) {
2708         my $st = stat($fd);
2709         my $line = <$fd>;
2710         close($fd);
2711
2712         my $mtime = $st->mtime;
2713         if ($mtime > time()) {
2714             warn "file '$filename' modified in future\n";
2715         }
2716
2717         if ($line =~ m/^(\d+)$/) {
2718             my $pid = $1;
2719             if (check_cmdline($pidfile, $pid)) {
2720                 if (my $pinfo = PVE::ProcFSTools::check_process_running($pid)) {
2721                     return $pid;
2722                 }
2723             }
2724         }
2725     }
2726
2727     return undef;
2728 }
2729
2730 sub vzlist {
2731
2732     my $vzlist = config_list();
2733
2734     my $fd = IO::Dir->new($var_run_tmpdir) || return $vzlist;
2735
2736     while (defined(my $de = $fd->read)) {
2737         next if $de !~ m/^(\d+)\.pid$/;
2738         my $vmid = $1;
2739         next if !defined($vzlist->{$vmid});
2740         if (my $pid = check_running($vmid)) {
2741             $vzlist->{$vmid}->{pid} = $pid;
2742         }
2743     }
2744
2745     return $vzlist;
2746 }
2747
2748 sub disksize {
2749     my ($storecfg, $conf) = @_;
2750
2751     my $bootdisk = $conf->{bootdisk};
2752     return undef if !$bootdisk;
2753     return undef if !is_valid_drivename($bootdisk);
2754
2755     return undef if !$conf->{$bootdisk};
2756
2757     my $drive = parse_drive($bootdisk, $conf->{$bootdisk});
2758     return undef if !defined($drive);
2759
2760     return undef if drive_is_cdrom($drive);
2761
2762     my $volid = $drive->{file};
2763     return undef if !$volid;
2764
2765     return $drive->{size};
2766 }
2767
2768 my $last_proc_pid_stat;
2769
2770 # get VM status information
2771 # This must be fast and should not block ($full == false)
2772 # We only query KVM using QMP if $full == true (this can be slow)
2773 sub vmstatus {
2774     my ($opt_vmid, $full) = @_;
2775
2776     my $res = {};
2777
2778     my $storecfg = PVE::Storage::config();
2779
2780     my $list = vzlist();
2781     my $defaults = load_defaults();
2782
2783     my ($uptime) = PVE::ProcFSTools::read_proc_uptime(1);
2784
2785     my $cpucount = $cpuinfo->{cpus} || 1;
2786
2787     foreach my $vmid (keys %$list) {
2788         next if $opt_vmid && ($vmid ne $opt_vmid);
2789
2790         my $cfspath = PVE::QemuConfig->cfs_config_path($vmid);
2791         my $conf = PVE::Cluster::cfs_read_file($cfspath) || {};
2792
2793         my $d = {};
2794         $d->{pid} = $list->{$vmid}->{pid};
2795
2796         # fixme: better status?
2797         $d->{status} = $list->{$vmid}->{pid} ? 'running' : 'stopped';
2798
2799         my $size = disksize($storecfg, $conf);
2800         if (defined($size)) {
2801             $d->{disk} = 0; # no info available
2802             $d->{maxdisk} = $size;
2803         } else {
2804             $d->{disk} = 0;
2805             $d->{maxdisk} = 0;
2806         }
2807
2808         $d->{cpus} = ($conf->{sockets} || $defaults->{sockets})
2809             * ($conf->{cores} || $defaults->{cores});
2810         $d->{cpus} = $cpucount if $d->{cpus} > $cpucount;
2811         $d->{cpus} = $conf->{vcpus} if $conf->{vcpus};
2812
2813         $d->{name} = $conf->{name} || "VM $vmid";
2814         $d->{maxmem} = $conf->{memory} ? $conf->{memory}*(1024*1024)
2815             : $defaults->{memory}*(1024*1024);
2816
2817         if ($conf->{balloon}) {
2818             $d->{balloon_min} = $conf->{balloon}*(1024*1024);
2819             $d->{shares} = defined($conf->{shares}) ? $conf->{shares}
2820                 : $defaults->{shares};
2821         }
2822
2823         $d->{uptime} = 0;
2824         $d->{cpu} = 0;
2825         $d->{mem} = 0;
2826
2827         $d->{netout} = 0;
2828         $d->{netin} = 0;
2829
2830         $d->{diskread} = 0;
2831         $d->{diskwrite} = 0;
2832
2833         $d->{template} = PVE::QemuConfig->is_template($conf);
2834
2835         $d->{serial} = 1 if conf_has_serial($conf);
2836
2837         $res->{$vmid} = $d;
2838     }
2839
2840     my $netdev = PVE::ProcFSTools::read_proc_net_dev();
2841     foreach my $dev (keys %$netdev) {
2842         next if $dev !~ m/^tap([1-9]\d*)i/;
2843         my $vmid = $1;
2844         my $d = $res->{$vmid};
2845         next if !$d;
2846
2847         $d->{netout} += $netdev->{$dev}->{receive};
2848         $d->{netin} += $netdev->{$dev}->{transmit};
2849
2850         if ($full) {
2851             $d->{nics}->{$dev}->{netout} = $netdev->{$dev}->{receive};
2852             $d->{nics}->{$dev}->{netin} = $netdev->{$dev}->{transmit};
2853         }
2854
2855     }
2856
2857     my $ctime = gettimeofday;
2858
2859     foreach my $vmid (keys %$list) {
2860
2861         my $d = $res->{$vmid};
2862         my $pid = $d->{pid};
2863         next if !$pid;
2864
2865         my $pstat = PVE::ProcFSTools::read_proc_pid_stat($pid);
2866         next if !$pstat; # not running
2867
2868         my $used = $pstat->{utime} + $pstat->{stime};
2869
2870         $d->{uptime} = int(($uptime - $pstat->{starttime})/$cpuinfo->{user_hz});
2871
2872         if ($pstat->{vsize}) {
2873             $d->{mem} = int(($pstat->{rss}/$pstat->{vsize})*$d->{maxmem});
2874         }
2875
2876         my $old = $last_proc_pid_stat->{$pid};
2877         if (!$old) {
2878             $last_proc_pid_stat->{$pid} = {
2879                 time => $ctime,
2880                 used => $used,
2881                 cpu => 0,
2882             };
2883             next;
2884         }
2885
2886         my $dtime = ($ctime -  $old->{time}) * $cpucount * $cpuinfo->{user_hz};
2887
2888         if ($dtime > 1000) {
2889             my $dutime = $used -  $old->{used};
2890
2891             $d->{cpu} = (($dutime/$dtime)* $cpucount) / $d->{cpus};
2892             $last_proc_pid_stat->{$pid} = {
2893                 time => $ctime,
2894                 used => $used,
2895                 cpu => $d->{cpu},
2896             };
2897         } else {
2898             $d->{cpu} = $old->{cpu};
2899         }
2900     }
2901
2902     return $res if !$full;
2903
2904     my $qmpclient = PVE::QMPClient->new();
2905
2906     my $ballooncb = sub {
2907         my ($vmid, $resp) = @_;
2908
2909         my $info = $resp->{'return'};
2910         return if !$info->{max_mem};
2911
2912         my $d = $res->{$vmid};
2913
2914         # use memory assigned to VM
2915         $d->{maxmem} = $info->{max_mem};
2916         $d->{balloon} = $info->{actual};
2917
2918         if (defined($info->{total_mem}) && defined($info->{free_mem})) {
2919             $d->{mem} = $info->{total_mem} - $info->{free_mem};
2920             $d->{freemem} = $info->{free_mem};
2921         }
2922
2923         $d->{ballooninfo} = $info;
2924     };
2925
2926     my $blockstatscb = sub {
2927         my ($vmid, $resp) = @_;
2928         my $data = $resp->{'return'} || [];
2929         my $totalrdbytes = 0;
2930         my $totalwrbytes = 0;
2931
2932         for my $blockstat (@$data) {
2933             $totalrdbytes = $totalrdbytes + $blockstat->{stats}->{rd_bytes};
2934             $totalwrbytes = $totalwrbytes + $blockstat->{stats}->{wr_bytes};
2935
2936             $blockstat->{device} =~ s/drive-//;
2937             $res->{$vmid}->{blockstat}->{$blockstat->{device}} = $blockstat->{stats};
2938         }
2939         $res->{$vmid}->{diskread} = $totalrdbytes;
2940         $res->{$vmid}->{diskwrite} = $totalwrbytes;
2941     };
2942
2943     my $statuscb = sub {
2944         my ($vmid, $resp) = @_;
2945
2946         $qmpclient->queue_cmd($vmid, $blockstatscb, 'query-blockstats');
2947         # this fails if ballon driver is not loaded, so this must be
2948         # the last commnand (following command are aborted if this fails).
2949         $qmpclient->queue_cmd($vmid, $ballooncb, 'query-balloon');
2950
2951         my $status = 'unknown';
2952         if (!defined($status = $resp->{'return'}->{status})) {
2953             warn "unable to get VM status\n";
2954             return;
2955         }
2956
2957         $res->{$vmid}->{qmpstatus} = $resp->{'return'}->{status};
2958     };
2959
2960     foreach my $vmid (keys %$list) {
2961         next if $opt_vmid && ($vmid ne $opt_vmid);
2962         next if !$res->{$vmid}->{pid}; # not running
2963         $qmpclient->queue_cmd($vmid, $statuscb, 'query-status');
2964     }
2965
2966     $qmpclient->queue_execute(undef, 2);
2967
2968     foreach my $vmid (keys %$list) {
2969         next if $opt_vmid && ($vmid ne $opt_vmid);
2970         $res->{$vmid}->{qmpstatus} = $res->{$vmid}->{status} if !$res->{$vmid}->{qmpstatus};
2971     }
2972
2973     return $res;
2974 }
2975
2976 sub foreach_drive {
2977     my ($conf, $func, @param) = @_;
2978
2979     foreach my $ds (valid_drive_names()) {
2980         next if !defined($conf->{$ds});
2981
2982         my $drive = parse_drive($ds, $conf->{$ds});
2983         next if !$drive;
2984
2985         &$func($ds, $drive, @param);
2986     }
2987 }
2988
2989 sub foreach_volid {
2990     my ($conf, $func, @param) = @_;
2991
2992     my $volhash = {};
2993
2994     my $test_volid = sub {
2995         my ($volid, $is_cdrom, $replicate, $shared, $snapname) = @_;
2996
2997         return if !$volid;
2998
2999         $volhash->{$volid}->{cdrom} //= 1;
3000         $volhash->{$volid}->{cdrom} = 0 if !$is_cdrom;
3001
3002         $volhash->{$volid}->{replicate} //= 0;
3003         $volhash->{$volid}->{replicate} = 1 if $replicate;
3004
3005         $volhash->{$volid}->{shared} //= 0;
3006         $volhash->{$volid}->{shared} = 1 if $shared;
3007
3008         $volhash->{$volid}->{referenced_in_config} //= 0;
3009         $volhash->{$volid}->{referenced_in_config} = 1 if !defined($snapname);
3010
3011         $volhash->{$volid}->{referenced_in_snapshot}->{$snapname} = 1
3012             if defined($snapname);
3013     };
3014
3015     foreach_drive($conf, sub {
3016         my ($ds, $drive) = @_;
3017         $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, undef);
3018     });
3019
3020     foreach my $snapname (keys %{$conf->{snapshots}}) {
3021         my $snap = $conf->{snapshots}->{$snapname};
3022         $test_volid->($snap->{vmstate}, 0, 1, $snapname);
3023         foreach_drive($snap, sub {
3024             my ($ds, $drive) = @_;
3025             $test_volid->($drive->{file}, drive_is_cdrom($drive), $drive->{replicate} // 1, $drive->{shared}, $snapname);
3026         });
3027     }
3028
3029     foreach my $volid (keys %$volhash) {
3030         &$func($volid, $volhash->{$volid}, @param);
3031     }
3032 }
3033
3034 sub conf_has_serial {
3035     my ($conf) = @_;
3036
3037     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3038         if ($conf->{"serial$i"}) {
3039             return 1;
3040         }
3041     }
3042
3043     return 0;
3044 }
3045
3046 sub vga_conf_has_spice {
3047     my ($vga) = @_;
3048
3049     return 0 if !$vga || $vga !~ m/^qxl([234])?$/;
3050
3051     return $1 || 1;
3052 }
3053
3054 sub config_to_command {
3055     my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
3056
3057     my $cmd = [];
3058     my $globalFlags = [];
3059     my $machineFlags = [];
3060     my $rtcFlags = [];
3061     my $cpuFlags = [];
3062     my $devices = [];
3063     my $pciaddr = '';
3064     my $bridges = {};
3065     my $kvmver = kvm_user_version();
3066     my $vernum = 0; # unknown
3067     my $ostype = $conf->{ostype};
3068     my $winversion = windows_version($ostype);
3069     my $kvm = $conf->{kvm} // 1;
3070
3071     die "KVM virtualisation configured, but not available. Either disable in VM configuration or enable in BIOS.\n" if (!$cpuinfo->{hvm} && $kvm);
3072
3073     if ($kvmver =~ m/^(\d+)\.(\d+)$/) {
3074         $vernum = $1*1000000+$2*1000;
3075     } elsif ($kvmver =~ m/^(\d+)\.(\d+)\.(\d+)$/) {
3076         $vernum = $1*1000000+$2*1000+$3;
3077     }
3078
3079     die "detected old qemu-kvm binary ($kvmver)\n" if $vernum < 15000;
3080
3081     my $have_ovz = -f '/proc/vz/vestat';
3082
3083     my $q35 = machine_type_is_q35($conf);
3084     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
3085     my $machine_type = $forcemachine || $conf->{machine};
3086     my $use_old_bios_files = undef;
3087     ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3088
3089     my $cpuunits = defined($conf->{cpuunits}) ?
3090             $conf->{cpuunits} : $defaults->{cpuunits};
3091
3092     push @$cmd, '/usr/bin/kvm';
3093
3094     push @$cmd, '-id', $vmid;
3095
3096     my $vmname = $conf->{name} || "vm$vmid";
3097
3098     push @$cmd, '-name', $vmname;
3099
3100     my $use_virtio = 0;
3101
3102     my $qmpsocket = qmp_socket($vmid);
3103     push @$cmd, '-chardev', "socket,id=qmp,path=$qmpsocket,server,nowait";
3104     push @$cmd, '-mon', "chardev=qmp,mode=control";
3105
3106
3107     push @$cmd, '-pidfile' , pidfile_name($vmid);
3108
3109     push @$cmd, '-daemonize';
3110
3111     if ($conf->{smbios1}) {
3112         push @$cmd, '-smbios', "type=1,$conf->{smbios1}";
3113     }
3114
3115     if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3116         die "uefi base image not found\n" if ! -f $OVMF_CODE;
3117
3118         my $path;
3119         my $format;
3120         if (my $efidisk = $conf->{efidisk0}) {
3121             my $d = PVE::JSONSchema::parse_property_string($efidisk_fmt, $efidisk);
3122             my ($storeid, $volname) = PVE::Storage::parse_volume_id($d->{file}, 1);
3123             $format = $d->{format};
3124             if ($storeid) {
3125                 $path = PVE::Storage::path($storecfg, $d->{file});
3126                 if (!defined($format)) {
3127                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
3128                     $format = qemu_img_format($scfg, $volname);
3129                 }
3130             } else {
3131                 $path = $d->{file};
3132                 die "efidisk format must be specified\n"
3133                     if !defined($format);
3134             }
3135         } else {
3136             warn "no efidisk configured! Using temporary efivars disk.\n";
3137             $path = "/tmp/$vmid-ovmf.fd";
3138             PVE::Tools::file_copy($OVMF_VARS, $path, -s $OVMF_VARS);
3139             $format = 'raw';
3140         }
3141
3142         push @$cmd, '-drive', "if=pflash,unit=0,format=raw,readonly,file=$OVMF_CODE";
3143         push @$cmd, '-drive', "if=pflash,unit=1,format=$format,id=drive-efidisk0,file=$path";
3144     }
3145
3146
3147     # add usb controllers
3148     my @usbcontrollers = PVE::QemuServer::USB::get_usb_controllers($conf, $bridges, $q35, $usbdesc->{format}, $MAX_USB_DEVICES);
3149     push @$devices, @usbcontrollers if @usbcontrollers;
3150     my $vga = $conf->{vga};
3151
3152     my $qxlnum = vga_conf_has_spice($vga);
3153     $vga = 'qxl' if $qxlnum;
3154
3155     if (!$vga) {
3156         if (qemu_machine_feature_enabled($machine_type, $kvmver, 2, 9)) {
3157             $vga = (!$winversion || $winversion >= 6) ? 'std' : 'cirrus';
3158         } else {
3159             $vga = ($winversion >= 6) ? 'std' : 'cirrus';
3160         }
3161     }
3162
3163     # enable absolute mouse coordinates (needed by vnc)
3164     my $tablet;
3165     if (defined($conf->{tablet})) {
3166         $tablet = $conf->{tablet};
3167     } else {
3168         $tablet = $defaults->{tablet};
3169         $tablet = 0 if $qxlnum; # disable for spice because it is not needed
3170         $tablet = 0 if $vga =~ m/^serial\d+$/; # disable if we use serial terminal (no vga card)
3171     }
3172
3173     push @$devices, '-device', print_tabletdevice_full($conf) if $tablet;
3174
3175     my $kvm_off = 0;
3176     my $gpu_passthrough;
3177
3178     # host pci devices
3179     for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
3180         my $d = parse_hostpci($conf->{"hostpci$i"});
3181         next if !$d;
3182
3183         my $pcie = $d->{pcie};
3184         if($pcie){
3185             die "q35 machine model is not enabled" if !$q35;
3186             $pciaddr = print_pcie_addr("hostpci$i");
3187         }else{
3188             $pciaddr = print_pci_addr("hostpci$i", $bridges);
3189         }
3190
3191         my $rombar = defined($d->{rombar}) && !$d->{rombar} ? ',rombar=0' : '';
3192         my $romfile = $d->{romfile};
3193
3194         my $xvga = '';
3195         if ($d->{'x-vga'}) {
3196             $xvga = ',x-vga=on';
3197             $kvm_off = 1;
3198             $vga = 'none';
3199             $gpu_passthrough = 1;
3200
3201             if ($conf->{bios} && $conf->{bios} eq 'ovmf') {
3202                 $xvga = "";
3203             }
3204         }
3205         my $pcidevices = $d->{pciid};
3206         my $multifunction = 1 if @$pcidevices > 1;
3207
3208         my $j=0;
3209         foreach my $pcidevice (@$pcidevices) {
3210
3211             my $id = "hostpci$i";
3212             $id .= ".$j" if $multifunction;
3213             my $addr = $pciaddr;
3214             $addr .= ".$j" if $multifunction;
3215             my $devicestr = "vfio-pci,host=$pcidevice->{id}.$pcidevice->{function},id=$id$addr";
3216
3217             if($j == 0){
3218                 $devicestr .= "$rombar$xvga";
3219                 $devicestr .= ",multifunction=on" if $multifunction;
3220                 $devicestr .= ",romfile=/usr/share/kvm/$romfile" if $romfile;
3221             }
3222
3223             push @$devices, '-device', $devicestr;
3224             $j++;
3225         }
3226     }
3227
3228     # usb devices
3229     my @usbdevices = PVE::QemuServer::USB::get_usb_devices($conf, $usbdesc->{format}, $MAX_USB_DEVICES);
3230     push @$devices, @usbdevices if @usbdevices;
3231     # serial devices
3232     for (my $i = 0; $i < $MAX_SERIAL_PORTS; $i++)  {
3233         if (my $path = $conf->{"serial$i"}) {
3234             if ($path eq 'socket') {
3235                 my $socket = "/var/run/qemu-server/${vmid}.serial$i";
3236                 push @$devices, '-chardev', "socket,id=serial$i,path=$socket,server,nowait";
3237                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3238             } else {
3239                 die "no such serial device\n" if ! -c $path;
3240                 push @$devices, '-chardev', "tty,id=serial$i,path=$path";
3241                 push @$devices, '-device', "isa-serial,chardev=serial$i";
3242             }
3243         }
3244     }
3245
3246     # parallel devices
3247     for (my $i = 0; $i < $MAX_PARALLEL_PORTS; $i++)  {
3248         if (my $path = $conf->{"parallel$i"}) {
3249             die "no such parallel device\n" if ! -c $path;
3250             my $devtype = $path =~ m!^/dev/usb/lp! ? 'tty' : 'parport';
3251             push @$devices, '-chardev', "$devtype,id=parallel$i,path=$path";
3252             push @$devices, '-device', "isa-parallel,chardev=parallel$i";
3253         }
3254     }
3255
3256
3257     my $sockets = 1;
3258     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
3259     $sockets = $conf->{sockets} if  $conf->{sockets};
3260
3261     my $cores = $conf->{cores} || 1;
3262
3263     my $maxcpus = $sockets * $cores;
3264
3265     my $vcpus = $conf->{vcpus} ? $conf->{vcpus} : $maxcpus;
3266
3267     my $allowed_vcpus = $cpuinfo->{cpus};
3268
3269     die "MAX $allowed_vcpus vcpus allowed per VM on this node\n"
3270         if ($allowed_vcpus < $maxcpus);
3271
3272     if($hotplug_features->{cpu} && qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 7)) {
3273
3274         push @$cmd, '-smp', "1,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3275         for (my $i = 2; $i <= $vcpus; $i++)  {
3276             my $cpustr = print_cpu_device($conf,$i);
3277             push @$cmd, '-device', $cpustr;
3278         }
3279
3280     } else {
3281
3282         push @$cmd, '-smp', "$vcpus,sockets=$sockets,cores=$cores,maxcpus=$maxcpus";
3283     }
3284     push @$cmd, '-nodefaults';
3285
3286     my $bootorder = $conf->{boot} || $confdesc->{boot}->{default};
3287
3288     my $bootindex_hash = {};
3289     my $i = 1;
3290     foreach my $o (split(//, $bootorder)) {
3291         $bootindex_hash->{$o} = $i*100;
3292         $i++;
3293     }
3294
3295     push @$cmd, '-boot', "menu=on,strict=on,reboot-timeout=1000,splash=/usr/share/qemu-server/bootsplash.jpg";
3296
3297     push @$cmd, '-no-acpi' if defined($conf->{acpi}) && $conf->{acpi} == 0;
3298
3299     push @$cmd, '-no-reboot' if  defined($conf->{reboot}) && $conf->{reboot} == 0;
3300
3301     push @$cmd, '-vga', $vga if $vga && $vga !~ m/^serial\d+$/; # for kvm 77 and later
3302
3303     if ($vga && $vga !~ m/^serial\d+$/ && $vga ne 'none'){
3304         my $socket = vnc_socket($vmid);
3305         push @$cmd,  '-vnc', "unix:$socket,x509,password";
3306     } else {
3307         push @$cmd, '-nographic';
3308     }
3309
3310     # time drift fix
3311     my $tdf = defined($conf->{tdf}) ? $conf->{tdf} : $defaults->{tdf};
3312
3313     my $useLocaltime = $conf->{localtime};
3314
3315     if ($winversion >= 5) { # windows
3316         $useLocaltime = 1 if !defined($conf->{localtime});
3317
3318         # use time drift fix when acpi is enabled
3319         if (!(defined($conf->{acpi}) && $conf->{acpi} == 0)) {
3320             $tdf = 1 if !defined($conf->{tdf});
3321         }
3322     }
3323
3324     if ($winversion >= 6) {
3325         push @$globalFlags, 'kvm-pit.lost_tick_policy=discard';
3326         push @$cmd, '-no-hpet';
3327     }
3328
3329     push @$rtcFlags, 'driftfix=slew' if $tdf;
3330
3331     if (!$kvm) {
3332         push @$machineFlags, 'accel=tcg';
3333     }
3334
3335     if ($machine_type) {
3336         push @$machineFlags, "type=${machine_type}";
3337     }
3338
3339     if ($conf->{startdate}) {
3340         push @$rtcFlags, "base=$conf->{startdate}";
3341     } elsif ($useLocaltime) {
3342         push @$rtcFlags, 'base=localtime';
3343     }
3344
3345     my $cpu = $kvm ? "kvm64" : "qemu64";
3346     if (my $cputype = $conf->{cpu}) {
3347         my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
3348             or die "Cannot parse cpu description: $cputype\n";
3349         $cpu = $cpuconf->{cputype};
3350         $kvm_off = 1 if $cpuconf->{hidden};
3351
3352         if (defined(my $flags = $cpuconf->{flags})) {
3353             push @$cpuFlags, split(";", $flags);
3354         }
3355     }
3356
3357     push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64';
3358
3359     push @$cpuFlags , '-x2apic'
3360         if $conf->{ostype} && $conf->{ostype} eq 'solaris';
3361
3362     push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
3363
3364     push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
3365
3366     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3367
3368         push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
3369         push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
3370     }
3371
3372     add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type, $kvmver, $conf->{bios}, $gpu_passthrough) if $kvm;
3373
3374     push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm;
3375
3376     push @$cpuFlags, 'kvm=off' if $kvm_off;
3377
3378     my $cpu_vendor = $cpu_vendor_list->{$cpu} ||
3379         die "internal error"; # should not happen
3380
3381     push @$cpuFlags, "vendor=${cpu_vendor}"
3382         if $cpu_vendor ne 'default';
3383
3384     $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
3385
3386     push @$cmd, '-cpu', $cpu;
3387
3388     PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores, $defaults, $hotplug_features, $cmd);
3389
3390     push @$cmd, '-S' if $conf->{freeze};
3391
3392     # set keyboard layout
3393     my $kb = $conf->{keyboard} || $defaults->{keyboard};
3394     push @$cmd, '-k', $kb if $kb;
3395
3396     # enable sound
3397     #my $soundhw = $conf->{soundhw} || $defaults->{soundhw};
3398     #push @$cmd, '-soundhw', 'es1370';
3399     #push @$cmd, '-soundhw', $soundhw if $soundhw;
3400
3401     if($conf->{agent}) {
3402         my $qgasocket = qmp_socket($vmid, 1);
3403         my $pciaddr = print_pci_addr("qga0", $bridges);
3404         push @$devices, '-chardev', "socket,path=$qgasocket,server,nowait,id=qga0";
3405         push @$devices, '-device', "virtio-serial,id=qga0$pciaddr";
3406         push @$devices, '-device', 'virtserialport,chardev=qga0,name=org.qemu.guest_agent.0';
3407     }
3408
3409     my $spice_port;
3410
3411     if ($qxlnum) {
3412         if ($qxlnum > 1) {
3413             if ($winversion){
3414                 for(my $i = 1; $i < $qxlnum; $i++){
3415                     my $pciaddr = print_pci_addr("vga$i", $bridges);
3416                     push @$cmd, '-device', "qxl,id=vga$i,ram_size=67108864,vram_size=33554432$pciaddr";
3417                 }
3418             } else {
3419                 # assume other OS works like Linux
3420                 push @$cmd, '-global', 'qxl-vga.ram_size=134217728';
3421                 push @$cmd, '-global', 'qxl-vga.vram_size=67108864';
3422             }
3423         }
3424
3425         my $pciaddr = print_pci_addr("spice", $bridges);
3426
3427         my $nodename = PVE::INotify::nodename();
3428         my $pfamily = PVE::Tools::get_host_address_family($nodename);
3429         my @nodeaddrs = PVE::Tools::getaddrinfo_all('localhost', family => $pfamily);
3430         die "failed to get an ip address of type $pfamily for 'localhost'\n" if !@nodeaddrs;
3431         my $localhost = PVE::Network::addr_to_ip($nodeaddrs[0]->{addr});
3432         $spice_port = PVE::Tools::next_spice_port($pfamily, $localhost);
3433
3434         push @$devices, '-spice', "tls-port=${spice_port},addr=$localhost,tls-ciphers=HIGH,seamless-migration=on";
3435
3436         push @$devices, '-device', "virtio-serial,id=spice$pciaddr";
3437         push @$devices, '-chardev', "spicevmc,id=vdagent,name=vdagent";
3438         push @$devices, '-device', "virtserialport,chardev=vdagent,name=com.redhat.spice.0";
3439     }
3440
3441     # enable balloon by default, unless explicitly disabled
3442     if (!defined($conf->{balloon}) || $conf->{balloon}) {
3443         $pciaddr = print_pci_addr("balloon0", $bridges);
3444         push @$devices, '-device', "virtio-balloon-pci,id=balloon0$pciaddr";
3445     }
3446
3447     if ($conf->{watchdog}) {
3448         my $wdopts = parse_watchdog($conf->{watchdog});
3449         $pciaddr = print_pci_addr("watchdog", $bridges);
3450         my $watchdog = $wdopts->{model} || 'i6300esb';
3451         push @$devices, '-device', "$watchdog$pciaddr";
3452         push @$devices, '-watchdog-action', $wdopts->{action} if $wdopts->{action};
3453     }
3454
3455     my $vollist = [];
3456     my $scsicontroller = {};
3457     my $ahcicontroller = {};
3458     my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : $defaults->{scsihw};
3459
3460     # Add iscsi initiator name if available
3461     if (my $initiator = get_initiator_name()) {
3462         push @$devices, '-iscsi', "initiator-name=$initiator";
3463     }
3464
3465     foreach_drive($conf, sub {
3466         my ($ds, $drive) = @_;
3467
3468         if (PVE::Storage::parse_volume_id($drive->{file}, 1)) {
3469             push @$vollist, $drive->{file};
3470         }
3471
3472         # ignore efidisk here, already added in bios/fw handling code above
3473         return if $drive->{interface} eq 'efidisk';
3474
3475         $use_virtio = 1 if $ds =~ m/^virtio/;
3476
3477         if (drive_is_cdrom ($drive)) {
3478             if ($bootindex_hash->{d}) {
3479                 $drive->{bootindex} = $bootindex_hash->{d};
3480                 $bootindex_hash->{d} += 1;
3481             }
3482         } else {
3483             if ($bootindex_hash->{c}) {
3484                 $drive->{bootindex} = $bootindex_hash->{c} if $conf->{bootdisk} && ($conf->{bootdisk} eq $ds);
3485                 $bootindex_hash->{c} += 1;
3486             }
3487         }
3488
3489         if($drive->{interface} eq 'virtio'){
3490            push @$cmd, '-object', "iothread,id=iothread-$ds" if $drive->{iothread};
3491         }
3492
3493         if ($drive->{interface} eq 'scsi') {
3494
3495             my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $drive);
3496
3497             $pciaddr = print_pci_addr("$controller_prefix$controller", $bridges);
3498             my $scsihw_type = $scsihw =~ m/^virtio-scsi-single/ ? "virtio-scsi-pci" : $scsihw;
3499
3500             my $iothread = '';
3501             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{iothread}){
3502                 $iothread .= ",iothread=iothread-$controller_prefix$controller";
3503                 push @$cmd, '-object', "iothread,id=iothread-$controller_prefix$controller";
3504             } elsif ($drive->{iothread}) {
3505                 warn "iothread is only valid with virtio disk or virtio-scsi-single controller, ignoring\n";
3506             }
3507
3508             my $queues = '';
3509             if($conf->{scsihw} && $conf->{scsihw} eq "virtio-scsi-single" && $drive->{queues}){
3510                 $queues = ",num_queues=$drive->{queues}";
3511             }
3512
3513             push @$devices, '-device', "$scsihw_type,id=$controller_prefix$controller$pciaddr$iothread$queues" if !$scsicontroller->{$controller};
3514             $scsicontroller->{$controller}=1;
3515         }
3516
3517         if ($drive->{interface} eq 'sata') {
3518            my $controller = int($drive->{index} / $MAX_SATA_DISKS);
3519            $pciaddr = print_pci_addr("ahci$controller", $bridges);
3520            push @$devices, '-device', "ahci,id=ahci$controller,multifunction=on$pciaddr" if !$ahcicontroller->{$controller};
3521            $ahcicontroller->{$controller}=1;
3522         }
3523
3524         my $drive_cmd = print_drive_full($storecfg, $vmid, $drive);
3525         push @$devices, '-drive',$drive_cmd;
3526         push @$devices, '-device', print_drivedevice_full($storecfg, $conf, $vmid, $drive, $bridges);
3527     });
3528
3529     for (my $i = 0; $i < $MAX_NETS; $i++) {
3530          next if !$conf->{"net$i"};
3531          my $d = parse_net($conf->{"net$i"});
3532          next if !$d;
3533
3534          $use_virtio = 1 if $d->{model} eq 'virtio';
3535
3536          if ($bootindex_hash->{n}) {
3537             $d->{bootindex} = $bootindex_hash->{n};
3538             $bootindex_hash->{n} += 1;
3539          }
3540
3541          my $netdevfull = print_netdev_full($vmid,$conf,$d,"net$i");
3542          push @$devices, '-netdev', $netdevfull;
3543
3544          my $netdevicefull = print_netdevice_full($vmid, $conf, $d, "net$i", $bridges, $use_old_bios_files);
3545          push @$devices, '-device', $netdevicefull;
3546     }
3547
3548     if (!$q35) {
3549         # add pci bridges
3550         if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
3551            $bridges->{1} = 1;
3552            $bridges->{2} = 1;
3553         }
3554
3555         $bridges->{3} = 1 if $scsihw =~ m/^virtio-scsi-single/;
3556
3557         while (my ($k, $v) = each %$bridges) {
3558             $pciaddr = print_pci_addr("pci.$k");
3559             unshift @$devices, '-device', "pci-bridge,id=pci.$k,chassis_nr=$k$pciaddr" if $k > 0;
3560         }
3561     }
3562
3563     # add custom args
3564     if ($conf->{args}) {
3565         my $aa = PVE::Tools::split_args($conf->{args});
3566         push @$cmd, @$aa;
3567     }
3568
3569     push @$cmd, @$devices;
3570     push @$cmd, '-rtc', join(',', @$rtcFlags)
3571         if scalar(@$rtcFlags);
3572     push @$cmd, '-machine', join(',', @$machineFlags)
3573         if scalar(@$machineFlags);
3574     push @$cmd, '-global', join(',', @$globalFlags)
3575         if scalar(@$globalFlags);
3576
3577     return wantarray ? ($cmd, $vollist, $spice_port) : $cmd;
3578 }
3579
3580 sub vnc_socket {
3581     my ($vmid) = @_;
3582     return "${var_run_tmpdir}/$vmid.vnc";
3583 }
3584
3585 sub spice_port {
3586     my ($vmid) = @_;
3587
3588     my $res = vm_mon_cmd($vmid, 'query-spice');
3589
3590     return $res->{'tls-port'} || $res->{'port'} || die "no spice port\n";
3591 }
3592
3593 sub qmp_socket {
3594     my ($vmid, $qga) = @_;
3595     my $sockettype = $qga ? 'qga' : 'qmp';
3596     return "${var_run_tmpdir}/$vmid.$sockettype";
3597 }
3598
3599 sub pidfile_name {
3600     my ($vmid) = @_;
3601     return "${var_run_tmpdir}/$vmid.pid";
3602 }
3603
3604 sub vm_devices_list {
3605     my ($vmid) = @_;
3606
3607     my $res = vm_mon_cmd($vmid, 'query-pci');
3608     my $devices = {};
3609     foreach my $pcibus (@$res) {
3610         foreach my $device (@{$pcibus->{devices}}) {
3611             next if !$device->{'qdev_id'};
3612             if ($device->{'pci_bridge'}) {
3613                 $devices->{$device->{'qdev_id'}} = 1;
3614                 foreach my $bridge_device (@{$device->{'pci_bridge'}->{devices}}) {
3615                     next if !$bridge_device->{'qdev_id'};
3616                     $devices->{$bridge_device->{'qdev_id'}} = 1;
3617                     $devices->{$device->{'qdev_id'}}++;
3618                 }
3619             } else {
3620                 $devices->{$device->{'qdev_id'}} = 1;
3621             }
3622         }
3623     }
3624
3625     my $resblock = vm_mon_cmd($vmid, 'query-block');
3626     foreach my $block (@$resblock) {
3627         if($block->{device} =~ m/^drive-(\S+)/){
3628                 $devices->{$1} = 1;
3629         }
3630     }
3631
3632     my $resmice = vm_mon_cmd($vmid, 'query-mice');
3633     foreach my $mice (@$resmice) {
3634         if ($mice->{name} eq 'QEMU HID Tablet') {
3635             $devices->{tablet} = 1;
3636             last;
3637         }
3638     }
3639
3640     # for usb devices there is no query-usb
3641     # but we can iterate over the entries in
3642     # qom-list path=/machine/peripheral
3643     my $resperipheral = vm_mon_cmd($vmid, 'qom-list', path => '/machine/peripheral');
3644     foreach my $per (@$resperipheral) {
3645         if ($per->{name} =~ m/^usb\d+$/) {
3646             $devices->{$per->{name}} = 1;
3647         }
3648     }
3649
3650     return $devices;
3651 }
3652
3653 sub vm_deviceplug {
3654     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
3655
3656     my $q35 = machine_type_is_q35($conf);
3657
3658     my $devices_list = vm_devices_list($vmid);
3659     return 1 if defined($devices_list->{$deviceid});
3660
3661     qemu_add_pci_bridge($storecfg, $conf, $vmid, $deviceid); # add PCI bridge if we need it for the device
3662
3663     if ($deviceid eq 'tablet') {
3664
3665         qemu_deviceadd($vmid, print_tabletdevice_full($conf));
3666
3667     } elsif ($deviceid =~ m/^usb(\d+)$/) {
3668
3669         die "usb hotplug currently not reliable\n";
3670         # since we can't reliably hot unplug all added usb devices
3671         # and usb passthrough disables live migration
3672         # we disable usb hotplugging for now
3673         qemu_deviceadd($vmid, PVE::QemuServer::USB::print_usbdevice_full($conf, $deviceid, $device));
3674
3675     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3676
3677         qemu_iothread_add($vmid, $deviceid, $device);
3678
3679         qemu_driveadd($storecfg, $vmid, $device);
3680         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3681
3682         qemu_deviceadd($vmid, $devicefull);
3683         eval { qemu_deviceaddverify($vmid, $deviceid); };
3684         if (my $err = $@) {
3685             eval { qemu_drivedel($vmid, $deviceid); };
3686             warn $@ if $@;
3687             die $err;
3688         }
3689
3690     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3691
3692
3693         my $scsihw = defined($conf->{scsihw}) ? $conf->{scsihw} : "lsi";
3694         my $pciaddr = print_pci_addr($deviceid);
3695         my $scsihw_type = $scsihw eq 'virtio-scsi-single' ? "virtio-scsi-pci" : $scsihw;
3696
3697         my $devicefull = "$scsihw_type,id=$deviceid$pciaddr";
3698
3699         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{iothread}) {
3700             qemu_iothread_add($vmid, $deviceid, $device);
3701             $devicefull .= ",iothread=iothread-$deviceid";
3702         }
3703
3704         if($deviceid =~ m/^virtioscsi(\d+)$/ && $device->{queues}) {
3705             $devicefull .= ",num_queues=$device->{queues}";
3706         }
3707
3708         qemu_deviceadd($vmid, $devicefull);
3709         qemu_deviceaddverify($vmid, $deviceid);
3710
3711     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3712
3713         qemu_findorcreatescsihw($storecfg,$conf, $vmid, $device);
3714         qemu_driveadd($storecfg, $vmid, $device);
3715
3716         my $devicefull = print_drivedevice_full($storecfg, $conf, $vmid, $device);
3717         eval { qemu_deviceadd($vmid, $devicefull); };
3718         if (my $err = $@) {
3719             eval { qemu_drivedel($vmid, $deviceid); };
3720             warn $@ if $@;
3721             die $err;
3722         }
3723
3724     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3725
3726         return undef if !qemu_netdevadd($vmid, $conf, $device, $deviceid);
3727
3728         my $machine_type = PVE::QemuServer::qemu_machine_pxe($vmid, $conf);
3729         my $use_old_bios_files = undef;
3730         ($use_old_bios_files, $machine_type) = qemu_use_old_bios_files($machine_type);
3731
3732         my $netdevicefull = print_netdevice_full($vmid, $conf, $device, $deviceid, undef, $use_old_bios_files);
3733         qemu_deviceadd($vmid, $netdevicefull);
3734         eval { qemu_deviceaddverify($vmid, $deviceid); };
3735         if (my $err = $@) {
3736             eval { qemu_netdevdel($vmid, $deviceid); };
3737             warn $@ if $@;
3738             die $err;
3739         }
3740
3741     } elsif (!$q35 && $deviceid =~ m/^(pci\.)(\d+)$/) {
3742
3743         my $bridgeid = $2;
3744         my $pciaddr = print_pci_addr($deviceid);
3745         my $devicefull = "pci-bridge,id=pci.$bridgeid,chassis_nr=$bridgeid$pciaddr";
3746
3747         qemu_deviceadd($vmid, $devicefull);
3748         qemu_deviceaddverify($vmid, $deviceid);
3749
3750     } else {
3751         die "can't hotplug device '$deviceid'\n";
3752     }
3753
3754     return 1;
3755 }
3756
3757 # fixme: this should raise exceptions on error!
3758 sub vm_deviceunplug {
3759     my ($vmid, $conf, $deviceid) = @_;
3760
3761     my $devices_list = vm_devices_list($vmid);
3762     return 1 if !defined($devices_list->{$deviceid});
3763
3764     die "can't unplug bootdisk" if $conf->{bootdisk} && $conf->{bootdisk} eq $deviceid;
3765
3766     if ($deviceid eq 'tablet') {
3767
3768         qemu_devicedel($vmid, $deviceid);
3769
3770     } elsif ($deviceid =~ m/^usb\d+$/) {
3771
3772         die "usb hotplug currently not reliable\n";
3773         # when unplugging usb devices this way,
3774         # there may be remaining usb controllers/hubs
3775         # so we disable it for now
3776         qemu_devicedel($vmid, $deviceid);
3777         qemu_devicedelverify($vmid, $deviceid);
3778
3779     } elsif ($deviceid =~ m/^(virtio)(\d+)$/) {
3780
3781         qemu_devicedel($vmid, $deviceid);
3782         qemu_devicedelverify($vmid, $deviceid);
3783         qemu_drivedel($vmid, $deviceid);
3784         qemu_iothread_del($conf, $vmid, $deviceid);
3785
3786     } elsif ($deviceid =~ m/^(virtioscsi|scsihw)(\d+)$/) {
3787
3788         qemu_devicedel($vmid, $deviceid);
3789         qemu_devicedelverify($vmid, $deviceid);
3790         qemu_iothread_del($conf, $vmid, $deviceid);
3791
3792     } elsif ($deviceid =~ m/^(scsi)(\d+)$/) {
3793
3794         qemu_devicedel($vmid, $deviceid);
3795         qemu_drivedel($vmid, $deviceid);
3796         qemu_deletescsihw($conf, $vmid, $deviceid);
3797
3798     } elsif ($deviceid =~ m/^(net)(\d+)$/) {
3799
3800         qemu_devicedel($vmid, $deviceid);
3801         qemu_devicedelverify($vmid, $deviceid);
3802         qemu_netdevdel($vmid, $deviceid);
3803
3804     } else {
3805         die "can't unplug device '$deviceid'\n";
3806     }
3807
3808     return 1;
3809 }
3810
3811 sub qemu_deviceadd {
3812     my ($vmid, $devicefull) = @_;
3813
3814     $devicefull = "driver=".$devicefull;
3815     my %options =  split(/[=,]/, $devicefull);
3816
3817     vm_mon_cmd($vmid, "device_add" , %options);
3818 }
3819
3820 sub qemu_devicedel {
3821     my ($vmid, $deviceid) = @_;
3822
3823     my $ret = vm_mon_cmd($vmid, "device_del", id => $deviceid);
3824 }
3825
3826 sub qemu_iothread_add {
3827     my($vmid, $deviceid, $device) = @_;
3828
3829     if ($device->{iothread}) {
3830         my $iothreads = vm_iothreads_list($vmid);
3831         qemu_objectadd($vmid, "iothread-$deviceid", "iothread") if !$iothreads->{"iothread-$deviceid"};
3832     }
3833 }
3834
3835 sub qemu_iothread_del {
3836     my($conf, $vmid, $deviceid) = @_;
3837
3838     my $device = parse_drive($deviceid, $conf->{$deviceid});
3839     if ($device->{iothread}) {
3840         my $iothreads = vm_iothreads_list($vmid);
3841         qemu_objectdel($vmid, "iothread-$deviceid") if $iothreads->{"iothread-$deviceid"};
3842     }
3843 }
3844
3845 sub qemu_objectadd {
3846     my($vmid, $objectid, $qomtype) = @_;
3847
3848     vm_mon_cmd($vmid, "object-add", id => $objectid, "qom-type" => $qomtype);
3849
3850     return 1;
3851 }
3852
3853 sub qemu_objectdel {
3854     my($vmid, $objectid) = @_;
3855
3856     vm_mon_cmd($vmid, "object-del", id => $objectid);
3857
3858     return 1;
3859 }
3860
3861 sub qemu_driveadd {
3862     my ($storecfg, $vmid, $device) = @_;
3863
3864     my $drive = print_drive_full($storecfg, $vmid, $device);
3865     $drive =~ s/\\/\\\\/g;
3866     my $ret = vm_human_monitor_command($vmid, "drive_add auto \"$drive\"");
3867
3868     # If the command succeeds qemu prints: "OK"
3869     return 1 if $ret =~ m/OK/s;
3870
3871     die "adding drive failed: $ret\n";
3872 }
3873
3874 sub qemu_drivedel {
3875     my($vmid, $deviceid) = @_;
3876
3877     my $ret = vm_human_monitor_command($vmid, "drive_del drive-$deviceid");
3878     $ret =~ s/^\s+//;
3879
3880     return 1 if $ret eq "";
3881
3882     # NB: device not found errors mean the drive was auto-deleted and we ignore the error
3883     return 1 if $ret =~ m/Device \'.*?\' not found/s;
3884
3885     die "deleting drive $deviceid failed : $ret\n";
3886 }
3887
3888 sub qemu_deviceaddverify {
3889     my ($vmid, $deviceid) = @_;
3890
3891     for (my $i = 0; $i <= 5; $i++) {
3892          my $devices_list = vm_devices_list($vmid);
3893          return 1 if defined($devices_list->{$deviceid});
3894          sleep 1;
3895     }
3896
3897     die "error on hotplug device '$deviceid'\n";
3898 }
3899
3900
3901 sub qemu_devicedelverify {
3902     my ($vmid, $deviceid) = @_;
3903
3904     # need to verify that the device is correctly removed as device_del
3905     # is async and empty return is not reliable
3906
3907     for (my $i = 0; $i <= 5; $i++) {
3908          my $devices_list = vm_devices_list($vmid);
3909          return 1 if !defined($devices_list->{$deviceid});
3910          sleep 1;
3911     }
3912
3913     die "error on hot-unplugging device '$deviceid'\n";
3914 }
3915
3916 sub qemu_findorcreatescsihw {
3917     my ($storecfg, $conf, $vmid, $device) = @_;
3918
3919     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3920
3921     my $scsihwid="$controller_prefix$controller";
3922     my $devices_list = vm_devices_list($vmid);
3923
3924     if(!defined($devices_list->{$scsihwid})) {
3925         vm_deviceplug($storecfg, $conf, $vmid, $scsihwid, $device);
3926     }
3927
3928     return 1;
3929 }
3930
3931 sub qemu_deletescsihw {
3932     my ($conf, $vmid, $opt) = @_;
3933
3934     my $device = parse_drive($opt, $conf->{$opt});
3935
3936     if ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
3937         vm_deviceunplug($vmid, $conf, "virtioscsi$device->{index}");
3938         return 1;
3939     }
3940
3941     my ($maxdev, $controller, $controller_prefix) = scsihw_infos($conf, $device);
3942
3943     my $devices_list = vm_devices_list($vmid);
3944     foreach my $opt (keys %{$devices_list}) {
3945         if (PVE::QemuServer::is_valid_drivename($opt)) {
3946             my $drive = PVE::QemuServer::parse_drive($opt, $conf->{$opt});
3947             if($drive->{interface} eq 'scsi' && $drive->{index} < (($maxdev-1)*($controller+1))) {
3948                 return 1;
3949             }
3950         }
3951     }
3952
3953     my $scsihwid="scsihw$controller";
3954
3955     vm_deviceunplug($vmid, $conf, $scsihwid);
3956
3957     return 1;
3958 }
3959
3960 sub qemu_add_pci_bridge {
3961     my ($storecfg, $conf, $vmid, $device) = @_;
3962
3963     my $bridges = {};
3964
3965     my $bridgeid;
3966
3967     print_pci_addr($device, $bridges);
3968
3969     while (my ($k, $v) = each %$bridges) {
3970         $bridgeid = $k;
3971     }
3972     return 1 if !defined($bridgeid) || $bridgeid < 1;
3973
3974     my $bridge = "pci.$bridgeid";
3975     my $devices_list = vm_devices_list($vmid);
3976
3977     if (!defined($devices_list->{$bridge})) {
3978         vm_deviceplug($storecfg, $conf, $vmid, $bridge);
3979     }
3980
3981     return 1;
3982 }
3983
3984 sub qemu_set_link_status {
3985     my ($vmid, $device, $up) = @_;
3986
3987     vm_mon_cmd($vmid, "set_link", name => $device,
3988                up => $up ? JSON::true : JSON::false);
3989 }
3990
3991 sub qemu_netdevadd {
3992     my ($vmid, $conf, $device, $deviceid) = @_;
3993
3994     my $netdev = print_netdev_full($vmid, $conf, $device, $deviceid, 1);
3995     my %options =  split(/[=,]/, $netdev);
3996
3997     vm_mon_cmd($vmid, "netdev_add",  %options);
3998     return 1;
3999 }
4000
4001 sub qemu_netdevdel {
4002     my ($vmid, $deviceid) = @_;
4003
4004     vm_mon_cmd($vmid, "netdev_del", id => $deviceid);
4005 }
4006
4007 sub qemu_usb_hotplug {
4008     my ($storecfg, $conf, $vmid, $deviceid, $device) = @_;
4009
4010     return if !$device;
4011
4012     # remove the old one first
4013     vm_deviceunplug($vmid, $conf, $deviceid);
4014
4015     # check if xhci controller is necessary and available
4016     if ($device->{usb3}) {
4017
4018         my $devicelist = vm_devices_list($vmid);
4019
4020         if (!$devicelist->{xhci}) {
4021             my $pciaddr = print_pci_addr("xhci");
4022             qemu_deviceadd($vmid, "nec-usb-xhci,id=xhci$pciaddr");
4023         }
4024     }
4025     my $d = parse_usb_device($device->{host});
4026     $d->{usb3} = $device->{usb3};
4027
4028     # add the new one
4029     vm_deviceplug($storecfg, $conf, $vmid, $deviceid, $d);
4030 }
4031
4032 sub qemu_cpu_hotplug {
4033     my ($vmid, $conf, $vcpus) = @_;
4034
4035     my $machine_type = PVE::QemuServer::get_current_qemu_machine($vmid);
4036
4037     my $sockets = 1;
4038     $sockets = $conf->{smp} if $conf->{smp}; # old style - no longer iused
4039     $sockets = $conf->{sockets} if  $conf->{sockets};
4040     my $cores = $conf->{cores} || 1;
4041     my $maxcpus = $sockets * $cores;
4042
4043     $vcpus = $maxcpus if !$vcpus;
4044
4045     die "you can't add more vcpus than maxcpus\n"
4046         if $vcpus > $maxcpus;
4047
4048     my $currentvcpus = $conf->{vcpus} || $maxcpus;
4049
4050     if ($vcpus < $currentvcpus) {
4051
4052         if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4053
4054             for (my $i = $currentvcpus; $i > $vcpus; $i--) {
4055                 qemu_devicedel($vmid, "cpu$i");
4056                 my $retry = 0;
4057                 my $currentrunningvcpus = undef;
4058                 while (1) {
4059                     $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4060                     last if scalar(@{$currentrunningvcpus}) == $i-1;
4061                     raise_param_exc({ vcpus => "error unplugging cpu$i" }) if $retry > 5;
4062                     $retry++;
4063                     sleep 1;
4064                 }
4065                 #update conf after each succesfull cpu unplug
4066                 $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4067                 PVE::QemuConfig->write_config($vmid, $conf);
4068             }
4069         } else {
4070             die "cpu hot-unplugging requires qemu version 2.7 or higher\n";
4071         }
4072
4073         return;
4074     }
4075
4076     my $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4077     die "vcpus in running vm does not match its configuration\n"
4078         if scalar(@{$currentrunningvcpus}) != $currentvcpus;
4079
4080     if (qemu_machine_feature_enabled ($machine_type, undef, 2, 7)) {
4081
4082         for (my $i = $currentvcpus+1; $i <= $vcpus; $i++) {
4083             my $cpustr = print_cpu_device($conf, $i);
4084             qemu_deviceadd($vmid, $cpustr);
4085
4086             my $retry = 0;
4087             my $currentrunningvcpus = undef;
4088             while (1) {
4089                 $currentrunningvcpus = vm_mon_cmd($vmid, "query-cpus");
4090                 last if scalar(@{$currentrunningvcpus}) == $i;
4091                 raise_param_exc({ vcpus => "error hotplugging cpu$i" }) if $retry > 10;
4092                 sleep 1;
4093                 $retry++;
4094             }
4095             #update conf after each succesfull cpu hotplug
4096             $conf->{vcpus} = scalar(@{$currentrunningvcpus});
4097             PVE::QemuConfig->write_config($vmid, $conf);
4098         }
4099     } else {
4100
4101         for (my $i = $currentvcpus; $i < $vcpus; $i++) {
4102             vm_mon_cmd($vmid, "cpu-add", id => int($i));
4103         }
4104     }
4105 }
4106
4107 sub qemu_block_set_io_throttle {
4108     my ($vmid, $deviceid,
4109         $bps, $bps_rd, $bps_wr, $iops, $iops_rd, $iops_wr,
4110         $bps_max, $bps_rd_max, $bps_wr_max, $iops_max, $iops_rd_max, $iops_wr_max,
4111         $bps_max_length, $bps_rd_max_length, $bps_wr_max_length,
4112         $iops_max_length, $iops_rd_max_length, $iops_wr_max_length) = @_;
4113
4114     return if !check_running($vmid) ;
4115
4116     vm_mon_cmd($vmid, "block_set_io_throttle", device => $deviceid,
4117         bps => int($bps),
4118         bps_rd => int($bps_rd),
4119         bps_wr => int($bps_wr),
4120         iops => int($iops),
4121         iops_rd => int($iops_rd),
4122         iops_wr => int($iops_wr),
4123         bps_max => int($bps_max),
4124         bps_rd_max => int($bps_rd_max),
4125         bps_wr_max => int($bps_wr_max),
4126         iops_max => int($iops_max),
4127         iops_rd_max => int($iops_rd_max),
4128         iops_wr_max => int($iops_wr_max),
4129         bps_max_length => int($bps_max_length),
4130         bps_rd_max_length => int($bps_rd_max_length),
4131         bps_wr_max_length => int($bps_wr_max_length),
4132         iops_max_length => int($iops_max_length),
4133         iops_rd_max_length => int($iops_rd_max_length),
4134         iops_wr_max_length => int($iops_wr_max_length),
4135     );
4136
4137 }
4138
4139 # old code, only used to shutdown old VM after update
4140 sub __read_avail {
4141     my ($fh, $timeout) = @_;
4142
4143     my $sel = new IO::Select;
4144     $sel->add($fh);
4145
4146     my $res = '';
4147     my $buf;
4148
4149     my @ready;
4150     while (scalar (@ready = $sel->can_read($timeout))) {
4151         my $count;
4152         if ($count = $fh->sysread($buf, 8192)) {
4153             if ($buf =~ /^(.*)\(qemu\) $/s) {
4154                 $res .= $1;
4155                 last;
4156             } else {
4157                 $res .= $buf;
4158             }
4159         } else {
4160             if (!defined($count)) {
4161                 die "$!\n";
4162             }
4163             last;
4164         }
4165     }
4166
4167     die "monitor read timeout\n" if !scalar(@ready);
4168
4169     return $res;
4170 }
4171
4172 sub qemu_block_resize {
4173     my ($vmid, $deviceid, $storecfg, $volid, $size) = @_;
4174
4175     my $running = check_running($vmid);
4176
4177     $size = 0 if !PVE::Storage::volume_resize($storecfg, $volid, $size, $running);
4178
4179     return if !$running;
4180
4181     vm_mon_cmd($vmid, "block_resize", device => $deviceid, size => int($size));
4182
4183 }
4184
4185 sub qemu_volume_snapshot {
4186     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4187
4188     my $running = check_running($vmid);
4189
4190     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4191         vm_mon_cmd($vmid, "snapshot-drive", device => $deviceid, name => $snap);
4192     } else {
4193         PVE::Storage::volume_snapshot($storecfg, $volid, $snap);
4194     }
4195 }
4196
4197 sub qemu_volume_snapshot_delete {
4198     my ($vmid, $deviceid, $storecfg, $volid, $snap) = @_;
4199
4200     my $running = check_running($vmid);
4201
4202     if ($running && do_snapshots_with_qemu($storecfg, $volid)){
4203         vm_mon_cmd($vmid, "delete-drive-snapshot", device => $deviceid, name => $snap);
4204     } else {
4205         PVE::Storage::volume_snapshot_delete($storecfg, $volid, $snap, $running);
4206     }
4207 }
4208
4209 sub set_migration_caps {
4210     my ($vmid) = @_;
4211
4212     my $cap_ref = [];
4213
4214     my $enabled_cap = {
4215         "auto-converge" => 1,
4216         "xbzrle" => 1,
4217         "x-rdma-pin-all" => 0,
4218         "zero-blocks" => 0,
4219         "compress" => 0
4220     };
4221
4222     my $supported_capabilities = vm_mon_cmd_nocheck($vmid, "query-migrate-capabilities");
4223
4224     for my $supported_capability (@$supported_capabilities) {
4225         push @$cap_ref, {
4226             capability => $supported_capability->{capability},
4227             state => $enabled_cap->{$supported_capability->{capability}} ? JSON::true : JSON::false,
4228         };
4229     }
4230
4231     vm_mon_cmd_nocheck($vmid, "migrate-set-capabilities", capabilities => $cap_ref);
4232 }
4233
4234 my $fast_plug_option = {
4235     'lock' => 1,
4236     'name' => 1,
4237     'onboot' => 1,
4238     'shares' => 1,
4239     'startup' => 1,
4240     'description' => 1,
4241     'protection' => 1,
4242     'vmstatestorage' => 1,
4243 };
4244
4245 # hotplug changes in [PENDING]
4246 # $selection hash can be used to only apply specified options, for
4247 # example: { cores => 1 } (only apply changed 'cores')
4248 # $errors ref is used to return error messages
4249 sub vmconfig_hotplug_pending {
4250     my ($vmid, $conf, $storecfg, $selection, $errors) = @_;
4251
4252     my $defaults = load_defaults();
4253
4254     # commit values which do not have any impact on running VM first
4255     # Note: those option cannot raise errors, we we do not care about
4256     # $selection and always apply them.
4257
4258     my $add_error = sub {
4259         my ($opt, $msg) = @_;
4260         $errors->{$opt} = "hotplug problem - $msg";
4261     };
4262
4263     my $changes = 0;
4264     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4265         if ($fast_plug_option->{$opt}) {
4266             $conf->{$opt} = $conf->{pending}->{$opt};
4267             delete $conf->{pending}->{$opt};
4268             $changes = 1;
4269         }
4270     }
4271
4272     if ($changes) {
4273         PVE::QemuConfig->write_config($vmid, $conf);
4274         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4275     }
4276
4277     my $hotplug_features = parse_hotplug_features(defined($conf->{hotplug}) ? $conf->{hotplug} : '1');
4278
4279     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4280     while (my ($opt, $force) = each %$pending_delete_hash) {
4281         next if $selection && !$selection->{$opt};
4282         eval {
4283             if ($opt eq 'hotplug') {
4284                 die "skip\n" if ($conf->{hotplug} =~ /memory/);
4285             } elsif ($opt eq 'tablet') {
4286                 die "skip\n" if !$hotplug_features->{usb};
4287                 if ($defaults->{tablet}) {
4288                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4289                 } else {
4290                     vm_deviceunplug($vmid, $conf, $opt);
4291                 }
4292             } elsif ($opt =~ m/^usb\d+/) {
4293                 die "skip\n";
4294                 # since we cannot reliably hot unplug usb devices
4295                 # we are disabling it
4296                 die "skip\n" if !$hotplug_features->{usb} || $conf->{$opt} =~ m/spice/i;
4297                 vm_deviceunplug($vmid, $conf, $opt);
4298             } elsif ($opt eq 'vcpus') {
4299                 die "skip\n" if !$hotplug_features->{cpu};
4300                 qemu_cpu_hotplug($vmid, $conf, undef);
4301             } elsif ($opt eq 'balloon') {
4302                 # enable balloon device is not hotpluggable
4303                 die "skip\n" if !defined($conf->{balloon}) || $conf->{balloon};
4304             } elsif ($fast_plug_option->{$opt}) {
4305                 # do nothing
4306             } elsif ($opt =~ m/^net(\d+)$/) {
4307                 die "skip\n" if !$hotplug_features->{network};
4308                 vm_deviceunplug($vmid, $conf, $opt);
4309             } elsif (is_valid_drivename($opt)) {
4310                 die "skip\n" if !$hotplug_features->{disk} || $opt =~ m/(ide|sata)(\d+)/;
4311                 vm_deviceunplug($vmid, $conf, $opt);
4312                 vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4313             } elsif ($opt =~ m/^memory$/) {
4314                 die "skip\n" if !$hotplug_features->{memory};
4315                 PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt);
4316             } elsif ($opt eq 'cpuunits') {
4317                 cgroups_write("cpu", $vmid, "cpu.shares", $defaults->{cpuunits});
4318             } elsif ($opt eq 'cpulimit') {
4319                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", -1);
4320             } else {
4321                 die "skip\n";
4322             }
4323         };
4324         if (my $err = $@) {
4325             &$add_error($opt, $err) if $err ne "skip\n";
4326         } else {
4327             # save new config if hotplug was successful
4328             delete $conf->{$opt};
4329             vmconfig_undelete_pending_option($conf, $opt);
4330             PVE::QemuConfig->write_config($vmid, $conf);
4331             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4332         }
4333     }
4334
4335     my $apply_pending_cloudinit;
4336     $apply_pending_cloudinit = sub {
4337         my ($key, $value) = @_;
4338         $apply_pending_cloudinit = sub {}; # once is enough
4339
4340         my @cloudinit_opts = keys %$confdesc_cloudinit;
4341         foreach my $opt (keys %{$conf->{pending}}) {
4342             next if !grep { $_ eq $opt } @cloudinit_opts;
4343             $conf->{$opt} = delete $conf->{pending}->{$opt};
4344         }
4345
4346         my $new_conf = { %$conf };
4347         $new_conf->{$key} = $value;
4348         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($new_conf, $vmid);
4349     };
4350
4351     foreach my $opt (keys %{$conf->{pending}}) {
4352         next if $selection && !$selection->{$opt};
4353         my $value = $conf->{pending}->{$opt};
4354         eval {
4355             if ($opt eq 'hotplug') {
4356                 die "skip\n" if ($value =~ /memory/) || ($value !~ /memory/ && $conf->{hotplug} =~ /memory/);
4357             } elsif ($opt eq 'tablet') {
4358                 die "skip\n" if !$hotplug_features->{usb};
4359                 if ($value == 1) {
4360                     vm_deviceplug($storecfg, $conf, $vmid, $opt);
4361                 } elsif ($value == 0) {
4362                     vm_deviceunplug($vmid, $conf, $opt);
4363                 }
4364             } elsif ($opt =~ m/^usb\d+$/) {
4365                 die "skip\n";
4366                 # since we cannot reliably hot unplug usb devices
4367                 # we are disabling it
4368                 die "skip\n" if !$hotplug_features->{usb} || $value =~ m/spice/i;
4369                 my $d = eval { PVE::JSONSchema::parse_property_string($usbdesc->{format}, $value) };
4370                 die "skip\n" if !$d;
4371                 qemu_usb_hotplug($storecfg, $conf, $vmid, $opt, $d);
4372             } elsif ($opt eq 'vcpus') {
4373                 die "skip\n" if !$hotplug_features->{cpu};
4374                 qemu_cpu_hotplug($vmid, $conf, $value);
4375             } elsif ($opt eq 'balloon') {
4376                 # enable/disable balloning device is not hotpluggable
4377                 my $old_balloon_enabled =  !!(!defined($conf->{balloon}) || $conf->{balloon});
4378                 my $new_balloon_enabled =  !!(!defined($conf->{pending}->{balloon}) || $conf->{pending}->{balloon});
4379                 die "skip\n" if $old_balloon_enabled != $new_balloon_enabled;
4380
4381                 # allow manual ballooning if shares is set to zero
4382                 if ((defined($conf->{shares}) && ($conf->{shares} == 0))) {
4383                     my $balloon = $conf->{pending}->{balloon} || $conf->{memory} || $defaults->{memory};
4384                     vm_mon_cmd($vmid, "balloon", value => $balloon*1024*1024);
4385                 }
4386             } elsif ($opt =~ m/^net(\d+)$/) {
4387                 # some changes can be done without hotplug
4388                 vmconfig_update_net($storecfg, $conf, $hotplug_features->{network},
4389                                     $vmid, $opt, $value);
4390             } elsif (is_valid_drivename($opt)) {
4391                 # some changes can be done without hotplug
4392                 my $drive = parse_drive($opt, $value);
4393                 if (drive_is_cloudinit($drive)) {
4394                     &$apply_pending_cloudinit($opt, $value);
4395                 }
4396                 vmconfig_update_disk($storecfg, $conf, $hotplug_features->{disk},
4397                                      $vmid, $opt, $value, 1);
4398             } elsif ($opt =~ m/^memory$/) { #dimms
4399                 die "skip\n" if !$hotplug_features->{memory};
4400                 $value = PVE::QemuServer::Memory::qemu_memory_hotplug($vmid, $conf, $defaults, $opt, $value);
4401             } elsif ($opt eq 'cpuunits') {
4402                 cgroups_write("cpu", $vmid, "cpu.shares", $conf->{pending}->{$opt});
4403             } elsif ($opt eq 'cpulimit') {
4404                 my $cpulimit = $conf->{pending}->{$opt} == 0 ? -1 : int($conf->{pending}->{$opt} * 100000);
4405                 cgroups_write("cpu", $vmid, "cpu.cfs_quota_us", $cpulimit);
4406             } else {
4407                 die "skip\n";  # skip non-hot-pluggable options
4408             }
4409         };
4410         if (my $err = $@) {
4411             &$add_error($opt, $err) if $err ne "skip\n";
4412         } else {
4413             # save new config if hotplug was successful
4414             $conf->{$opt} = $value;
4415             delete $conf->{pending}->{$opt};
4416             PVE::QemuConfig->write_config($vmid, $conf);
4417             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4418         }
4419     }
4420 }
4421
4422 sub try_deallocate_drive {
4423     my ($storecfg, $vmid, $conf, $key, $drive, $rpcenv, $authuser, $force) = @_;
4424
4425     if (($force || $key =~ /^unused/) && !drive_is_cdrom($drive, 1)) {
4426         my $volid = $drive->{file};
4427         if (vm_is_volid_owner($storecfg, $vmid, $volid)) {
4428             my $sid = PVE::Storage::parse_volume_id($volid);
4429             $rpcenv->check($authuser, "/storage/$sid", ['Datastore.AllocateSpace']);
4430
4431             # check if the disk is really unused
4432             die "unable to delete '$volid' - volume is still in use (snapshot?)\n"
4433                 if is_volume_in_use($storecfg, $conf, $key, $volid);
4434             PVE::Storage::vdisk_free($storecfg, $volid);
4435             return 1;
4436         } else {
4437             # If vm is not owner of this disk remove from config
4438             return 1;
4439         }
4440     }
4441
4442     return undef;
4443 }
4444
4445 sub vmconfig_delete_or_detach_drive {
4446     my ($vmid, $storecfg, $conf, $opt, $force) = @_;
4447
4448     my $drive = parse_drive($opt, $conf->{$opt});
4449
4450     my $rpcenv = PVE::RPCEnvironment::get();
4451     my $authuser = $rpcenv->get_user();
4452
4453     if ($force) {
4454         $rpcenv->check_vm_perm($authuser, $vmid, undef, ['VM.Config.Disk']);
4455         try_deallocate_drive($storecfg, $vmid, $conf, $opt, $drive, $rpcenv, $authuser, $force);
4456     } else {
4457         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $drive);
4458     }
4459 }
4460
4461 sub vmconfig_apply_pending {
4462     my ($vmid, $conf, $storecfg) = @_;
4463
4464     # cold plug
4465
4466     my $pending_delete_hash = split_flagged_list($conf->{pending}->{delete});
4467     while (my ($opt, $force) = each %$pending_delete_hash) {
4468         die "internal error" if $opt =~ m/^unused/;
4469         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4470         if (!defined($conf->{$opt})) {
4471             vmconfig_undelete_pending_option($conf, $opt);
4472             PVE::QemuConfig->write_config($vmid, $conf);
4473         } elsif (is_valid_drivename($opt)) {
4474             vmconfig_delete_or_detach_drive($vmid, $storecfg, $conf, $opt, $force);
4475             vmconfig_undelete_pending_option($conf, $opt);
4476             delete $conf->{$opt};
4477             PVE::QemuConfig->write_config($vmid, $conf);
4478         } else {
4479             vmconfig_undelete_pending_option($conf, $opt);
4480             delete $conf->{$opt};
4481             PVE::QemuConfig->write_config($vmid, $conf);
4482         }
4483     }
4484
4485     $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4486
4487     foreach my $opt (keys %{$conf->{pending}}) { # add/change
4488         $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4489
4490         if (defined($conf->{$opt}) && ($conf->{$opt} eq $conf->{pending}->{$opt})) {
4491             # skip if nothing changed
4492         } elsif (is_valid_drivename($opt)) {
4493             vmconfig_register_unused_drive($storecfg, $vmid, $conf, parse_drive($opt, $conf->{$opt}))
4494                 if defined($conf->{$opt});
4495             $conf->{$opt} = $conf->{pending}->{$opt};
4496         } else {
4497             $conf->{$opt} = $conf->{pending}->{$opt};
4498         }
4499
4500         delete $conf->{pending}->{$opt};
4501         PVE::QemuConfig->write_config($vmid, $conf);
4502     }
4503 }
4504
4505 my $safe_num_ne = sub {
4506     my ($a, $b) = @_;
4507
4508     return 0 if !defined($a) && !defined($b);
4509     return 1 if !defined($a);
4510     return 1 if !defined($b);
4511
4512     return $a != $b;
4513 };
4514
4515 my $safe_string_ne = sub {
4516     my ($a, $b) = @_;
4517
4518     return 0 if !defined($a) && !defined($b);
4519     return 1 if !defined($a);
4520     return 1 if !defined($b);
4521
4522     return $a ne $b;
4523 };
4524
4525 sub vmconfig_update_net {
4526     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value) = @_;
4527
4528     my $newnet = parse_net($value);
4529
4530     if ($conf->{$opt}) {
4531         my $oldnet = parse_net($conf->{$opt});
4532
4533         if (&$safe_string_ne($oldnet->{model}, $newnet->{model}) ||
4534             &$safe_string_ne($oldnet->{macaddr}, $newnet->{macaddr}) ||
4535             &$safe_num_ne($oldnet->{queues}, $newnet->{queues}) ||
4536             !($newnet->{bridge} && $oldnet->{bridge})) { # bridge/nat mode change
4537
4538             # for non online change, we try to hot-unplug
4539             die "skip\n" if !$hotplug;
4540             vm_deviceunplug($vmid, $conf, $opt);
4541         } else {
4542
4543             die "internal error" if $opt !~ m/net(\d+)/;
4544             my $iface = "tap${vmid}i$1";
4545
4546             if (&$safe_string_ne($oldnet->{bridge}, $newnet->{bridge}) ||
4547                 &$safe_num_ne($oldnet->{tag}, $newnet->{tag}) ||
4548                 &$safe_string_ne($oldnet->{trunks}, $newnet->{trunks}) ||
4549                 &$safe_num_ne($oldnet->{firewall}, $newnet->{firewall})) {
4550                 PVE::Network::tap_unplug($iface);
4551                 PVE::Network::tap_plug($iface, $newnet->{bridge}, $newnet->{tag}, $newnet->{firewall}, $newnet->{trunks}, $newnet->{rate});
4552             } elsif (&$safe_num_ne($oldnet->{rate}, $newnet->{rate})) {
4553                 # Rate can be applied on its own but any change above needs to
4554                 # include the rate in tap_plug since OVS resets everything.
4555                 PVE::Network::tap_rate_limit($iface, $newnet->{rate});
4556             }
4557
4558             if (&$safe_string_ne($oldnet->{link_down}, $newnet->{link_down})) {
4559                 qemu_set_link_status($vmid, $opt, !$newnet->{link_down});
4560             }
4561
4562             return 1;
4563         }
4564     }
4565
4566     if ($hotplug) {
4567         vm_deviceplug($storecfg, $conf, $vmid, $opt, $newnet);
4568     } else {
4569         die "skip\n";
4570     }
4571 }
4572
4573 sub vmconfig_update_disk {
4574     my ($storecfg, $conf, $hotplug, $vmid, $opt, $value, $force) = @_;
4575
4576     # fixme: do we need force?
4577
4578     my $drive = parse_drive($opt, $value);
4579
4580     if ($conf->{$opt}) {
4581
4582         if (my $old_drive = parse_drive($opt, $conf->{$opt}))  {
4583
4584             my $media = $drive->{media} || 'disk';
4585             my $oldmedia = $old_drive->{media} || 'disk';
4586             die "unable to change media type\n" if $media ne $oldmedia;
4587
4588             if (!drive_is_cdrom($old_drive)) {
4589
4590                 if ($drive->{file} ne $old_drive->{file}) {
4591
4592                     die "skip\n" if !$hotplug;
4593
4594                     # unplug and register as unused
4595                     vm_deviceunplug($vmid, $conf, $opt);
4596                     vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive)
4597
4598                 } else {
4599                     # update existing disk
4600
4601                     # skip non hotpluggable value
4602                     if (&$safe_string_ne($drive->{discard}, $old_drive->{discard}) ||
4603                         &$safe_string_ne($drive->{iothread}, $old_drive->{iothread}) ||
4604                         &$safe_string_ne($drive->{queues}, $old_drive->{queues}) ||
4605                         &$safe_string_ne($drive->{cache}, $old_drive->{cache})) {
4606                         die "skip\n";
4607                     }
4608
4609                     # apply throttle
4610                     if (&$safe_num_ne($drive->{mbps}, $old_drive->{mbps}) ||
4611                         &$safe_num_ne($drive->{mbps_rd}, $old_drive->{mbps_rd}) ||
4612                         &$safe_num_ne($drive->{mbps_wr}, $old_drive->{mbps_wr}) ||
4613                         &$safe_num_ne($drive->{iops}, $old_drive->{iops}) ||
4614                         &$safe_num_ne($drive->{iops_rd}, $old_drive->{iops_rd}) ||
4615                         &$safe_num_ne($drive->{iops_wr}, $old_drive->{iops_wr}) ||
4616                         &$safe_num_ne($drive->{mbps_max}, $old_drive->{mbps_max}) ||
4617                         &$safe_num_ne($drive->{mbps_rd_max}, $old_drive->{mbps_rd_max}) ||
4618                         &$safe_num_ne($drive->{mbps_wr_max}, $old_drive->{mbps_wr_max}) ||
4619                         &$safe_num_ne($drive->{iops_max}, $old_drive->{iops_max}) ||
4620                         &$safe_num_ne($drive->{iops_rd_max}, $old_drive->{iops_rd_max}) ||
4621                         &$safe_num_ne($drive->{iops_wr_max}, $old_drive->{iops_wr_max}) ||
4622                         &$safe_num_ne($drive->{bps_max_length}, $old_drive->{bps_max_length}) ||
4623                         &$safe_num_ne($drive->{bps_rd_max_length}, $old_drive->{bps_rd_max_length}) ||
4624                         &$safe_num_ne($drive->{bps_wr_max_length}, $old_drive->{bps_wr_max_length}) ||
4625                         &$safe_num_ne($drive->{iops_max_length}, $old_drive->{iops_max_length}) ||
4626                         &$safe_num_ne($drive->{iops_rd_max_length}, $old_drive->{iops_rd_max_length}) ||
4627                         &$safe_num_ne($drive->{iops_wr_max_length}, $old_drive->{iops_wr_max_length})) {
4628
4629                         qemu_block_set_io_throttle($vmid,"drive-$opt",
4630                                                    ($drive->{mbps} || 0)*1024*1024,
4631                                                    ($drive->{mbps_rd} || 0)*1024*1024,
4632                                                    ($drive->{mbps_wr} || 0)*1024*1024,
4633                                                    $drive->{iops} || 0,
4634                                                    $drive->{iops_rd} || 0,
4635                                                    $drive->{iops_wr} || 0,
4636                                                    ($drive->{mbps_max} || 0)*1024*1024,
4637                                                    ($drive->{mbps_rd_max} || 0)*1024*1024,
4638                                                    ($drive->{mbps_wr_max} || 0)*1024*1024,
4639                                                    $drive->{iops_max} || 0,
4640                                                    $drive->{iops_rd_max} || 0,
4641                                                    $drive->{iops_wr_max} || 0,
4642                                                    $drive->{bps_max_length} || 1,
4643                                                    $drive->{bps_rd_max_length} || 1,
4644                                                    $drive->{bps_wr_max_length} || 1,
4645                                                    $drive->{iops_max_length} || 1,
4646                                                    $drive->{iops_rd_max_length} || 1,
4647                                                    $drive->{iops_wr_max_length} || 1);
4648
4649                     }
4650
4651                     return 1;
4652                 }
4653
4654             } else { # cdrom
4655
4656                 if ($drive->{file} eq 'none') {
4657                     vm_mon_cmd($vmid, "eject",force => JSON::true,device => "drive-$opt");
4658                     if (drive_is_cloudinit($old_drive)) {
4659                         vmconfig_register_unused_drive($storecfg, $vmid, $conf, $old_drive);
4660                     }
4661                 } else {
4662                     my $path = get_iso_path($storecfg, $vmid, $drive->{file});
4663                     vm_mon_cmd($vmid, "eject", force => JSON::true,device => "drive-$opt"); # force eject if locked
4664                     vm_mon_cmd($vmid, "change", device => "drive-$opt",target => "$path") if $path;
4665                 }
4666
4667                 return 1;
4668             }
4669         }
4670     }
4671
4672     die "skip\n" if !$hotplug || $opt =~ m/(ide|sata)(\d+)/;
4673     # hotplug new disks
4674     PVE::Storage::activate_volumes($storecfg, [$drive->{file}]) if $drive->{file} !~ m|^/dev/.+|;
4675     vm_deviceplug($storecfg, $conf, $vmid, $opt, $drive);
4676 }
4677
4678 sub vm_start {
4679     my ($storecfg, $vmid, $statefile, $skiplock, $migratedfrom, $paused,
4680         $forcemachine, $spice_ticket, $migration_network, $migration_type, $targetstorage) = @_;
4681
4682     PVE::QemuConfig->lock_config($vmid, sub {
4683         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
4684
4685         die "you can't start a vm if it's a template\n" if PVE::QemuConfig->is_template($conf);
4686
4687         PVE::QemuConfig->check_lock($conf) if !$skiplock;
4688
4689         die "VM $vmid already running\n" if check_running($vmid, undef, $migratedfrom);
4690
4691         if (!$statefile && scalar(keys %{$conf->{pending}})) {
4692             vmconfig_apply_pending($vmid, $conf, $storecfg);
4693             $conf = PVE::QemuConfig->load_config($vmid); # update/reload
4694         }
4695
4696         PVE::QemuServer::Cloudinit::generate_cloudinitconfig($conf, $vmid);
4697
4698         my $defaults = load_defaults();
4699
4700         # set environment variable useful inside network script
4701         $ENV{PVE_MIGRATED_FROM} = $migratedfrom if $migratedfrom;
4702
4703         my $local_volumes = {};
4704
4705         if ($targetstorage) {
4706             foreach_drive($conf, sub {
4707                 my ($ds, $drive) = @_;
4708
4709                 return if drive_is_cdrom($drive);
4710
4711                 my $volid = $drive->{file};
4712
4713                 return if !$volid;
4714
4715                 my ($storeid, $volname) = PVE::Storage::parse_volume_id($volid);
4716
4717                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4718                 return if $scfg->{shared};
4719                 $local_volumes->{$ds} = [$volid, $storeid, $volname];
4720             });
4721
4722             my $format = undef;
4723
4724             foreach my $opt (sort keys %$local_volumes) {
4725
4726                 my ($volid, $storeid, $volname) = @{$local_volumes->{$opt}};
4727                 my $drive = parse_drive($opt, $conf->{$opt});
4728
4729                 #if remote storage is specified, use default format
4730                 if ($targetstorage && $targetstorage ne "1") {
4731                     $storeid = $targetstorage;
4732                     my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
4733                     $format = $defFormat;
4734                 } else {
4735                     #else we use same format than original
4736                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
4737                     $format = qemu_img_format($scfg, $volid);
4738                 }
4739
4740                 my $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $format, undef, ($drive->{size}/1024));
4741                 my $newdrive = $drive;
4742                 $newdrive->{format} = $format;
4743                 $newdrive->{file} = $newvolid;
4744                 my $drivestr = PVE::QemuServer::print_drive($vmid, $newdrive);
4745                 $local_volumes->{$opt} = $drivestr;
4746                 #pass drive to conf for command line
4747                 $conf->{$opt} = $drivestr;
4748             }
4749         }
4750
4751         my ($cmd, $vollist, $spice_port) = config_to_command($storecfg, $vmid, $conf, $defaults, $forcemachine);
4752
4753         my $migrate_port = 0;
4754         my $migrate_uri;
4755         if ($statefile) {
4756             if ($statefile eq 'tcp') {
4757                 my $localip = "localhost";
4758                 my $datacenterconf = PVE::Cluster::cfs_read_file('datacenter.cfg');
4759                 my $nodename = PVE::INotify::nodename();
4760
4761                 if (!defined($migration_type)) {
4762                     if (defined($datacenterconf->{migration}->{type})) {
4763                         $migration_type = $datacenterconf->{migration}->{type};
4764                     } else {
4765                         $migration_type = 'secure';
4766                     }
4767                 }
4768
4769                 if ($migration_type eq 'insecure') {
4770                     my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4771                     if ($migrate_network_addr) {
4772                         $localip = $migrate_network_addr;
4773                     } else {
4774                         $localip = PVE::Cluster::remote_node_ip($nodename, 1);
4775                     }
4776
4777                     $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4778                 }
4779
4780                 my $pfamily = PVE::Tools::get_host_address_family($nodename);
4781                 $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4782                 $migrate_uri = "tcp:${localip}:${migrate_port}";
4783                 push @$cmd, '-incoming', $migrate_uri;
4784                 push @$cmd, '-S';
4785
4786             } elsif ($statefile eq 'unix') {
4787                 # should be default for secure migrations as a ssh TCP forward
4788                 # tunnel is not deterministic reliable ready and fails regurarly
4789                 # to set up in time, so use UNIX socket forwards
4790                 my $socket_addr = "/run/qemu-server/$vmid.migrate";
4791                 unlink $socket_addr;
4792
4793                 $migrate_uri = "unix:$socket_addr";
4794
4795                 push @$cmd, '-incoming', $migrate_uri;
4796                 push @$cmd, '-S';
4797
4798             } else {
4799                 push @$cmd, '-loadstate', $statefile;
4800             }
4801         } elsif ($paused) {
4802             push @$cmd, '-S';
4803         }
4804
4805         # host pci devices
4806         for (my $i = 0; $i < $MAX_HOSTPCI_DEVICES; $i++)  {
4807           my $d = parse_hostpci($conf->{"hostpci$i"});
4808           next if !$d;
4809           my $pcidevices = $d->{pciid};
4810           foreach my $pcidevice (@$pcidevices) {
4811                 my $pciid = $pcidevice->{id}.".".$pcidevice->{function};
4812
4813                 my $info = pci_device_info("0000:$pciid");
4814                 die "IOMMU not present\n" if !check_iommu_support();
4815                 die "no pci device info for device '$pciid'\n" if !$info;
4816                 die "can't unbind/bind pci group to vfio '$pciid'\n" if !pci_dev_group_bind_to_vfio($pciid);
4817                 die "can't reset pci device '$pciid'\n" if $info->{has_fl_reset} and !pci_dev_reset($info);
4818           }
4819         }
4820
4821         PVE::Storage::activate_volumes($storecfg, $vollist);
4822
4823         if (!check_running($vmid, 1)) {
4824             eval {
4825                 run_command(['/bin/systemctl', 'stop', "$vmid.scope"],
4826                     outfunc => sub {}, errfunc => sub {});
4827             };
4828         }
4829
4830         my $cpuunits = defined($conf->{cpuunits}) ? $conf->{cpuunits}
4831                                                   : $defaults->{cpuunits};
4832
4833         my $start_timeout = $conf->{hugepages} ? 300 : 30;
4834         my %run_params = (timeout => $statefile ? undef : $start_timeout, umask => 0077);
4835
4836         my %properties = (
4837             Slice => 'qemu.slice',
4838             KillMode => 'none',
4839             CPUShares => $cpuunits
4840         );
4841
4842         if (my $cpulimit = $conf->{cpulimit}) {
4843             $properties{CPUQuota} = int($cpulimit * 100);
4844         }
4845         $properties{timeout} = 10 if $statefile; # setting up the scope shoul be quick
4846
4847         if ($conf->{hugepages}) {
4848
4849             my $code = sub {
4850                 my $hugepages_topology = PVE::QemuServer::Memory::hugepages_topology($conf);
4851                 my $hugepages_host_topology = PVE::QemuServer::Memory::hugepages_host_topology();
4852
4853                 PVE::QemuServer::Memory::hugepages_mount();
4854                 PVE::QemuServer::Memory::hugepages_allocate($hugepages_topology, $hugepages_host_topology);
4855
4856                 eval  {
4857                     PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4858                     run_command($cmd, %run_params);
4859                 };
4860
4861                 if (my $err = $@) {
4862                     PVE::QemuServer::Memory::hugepages_reset($hugepages_host_topology);
4863                     die $err;
4864                 }
4865
4866                 PVE::QemuServer::Memory::hugepages_pre_deallocate($hugepages_topology);
4867             };
4868             eval { PVE::QemuServer::Memory::hugepages_update_locked($code); };
4869
4870         } else {
4871             eval  {
4872                 PVE::Tools::enter_systemd_scope($vmid, "Proxmox VE VM $vmid", %properties);
4873                 run_command($cmd, %run_params);
4874             };
4875         }
4876
4877         if (my $err = $@) {
4878             # deactivate volumes if start fails
4879             eval { PVE::Storage::deactivate_volumes($storecfg, $vollist); };
4880             die "start failed: $err";
4881         }
4882
4883         print "migration listens on $migrate_uri\n" if $migrate_uri;
4884
4885         if ($statefile && $statefile ne 'tcp' && $statefile ne 'unix')  {
4886             eval { vm_mon_cmd_nocheck($vmid, "cont"); };
4887             warn $@ if $@;
4888         }
4889
4890         #start nbd server for storage migration
4891         if ($targetstorage) {
4892             my $nodename = PVE::INotify::nodename();
4893             my $migrate_network_addr = PVE::Cluster::get_local_migration_ip($migration_network);
4894             my $localip = $migrate_network_addr ? $migrate_network_addr : PVE::Cluster::remote_node_ip($nodename, 1);
4895             my $pfamily = PVE::Tools::get_host_address_family($nodename);
4896             $migrate_port = PVE::Tools::next_migrate_port($pfamily);
4897
4898             vm_mon_cmd_nocheck($vmid, "nbd-server-start", addr => { type => 'inet', data => { host => "${localip}", port => "${migrate_port}" } } );
4899
4900             $localip = "[$localip]" if Net::IP::ip_is_ipv6($localip);
4901
4902             foreach my $opt (sort keys %$local_volumes) {
4903                 my $volid = $local_volumes->{$opt};
4904                 vm_mon_cmd_nocheck($vmid, "nbd-server-add", device => "drive-$opt", writable => JSON::true );
4905                 my $migrate_storage_uri = "nbd:${localip}:${migrate_port}:exportname=drive-$opt";
4906                 print "storage migration listens on $migrate_storage_uri volume:$volid\n";
4907             }
4908         }
4909
4910         if ($migratedfrom) {
4911             eval {
4912                 set_migration_caps($vmid);
4913             };
4914             warn $@ if $@;
4915
4916             if ($spice_port) {
4917                 print "spice listens on port $spice_port\n";
4918                 if ($spice_ticket) {
4919                     vm_mon_cmd_nocheck($vmid, "set_password", protocol => 'spice', password => $spice_ticket);
4920                     vm_mon_cmd_nocheck($vmid, "expire_password", protocol => 'spice', time => "+30");
4921                 }
4922             }
4923
4924         } else {
4925             if (!$statefile && (!defined($conf->{balloon}) || $conf->{balloon})) {
4926                 vm_mon_cmd_nocheck($vmid, "balloon", value => $conf->{balloon}*1024*1024)
4927                     if $conf->{balloon};
4928             }
4929
4930             foreach my $opt (keys %$conf) {
4931                 next if $opt !~  m/^net\d+$/;
4932                 my $nicconf = parse_net($conf->{$opt});
4933                 qemu_set_link_status($vmid, $opt, 0) if $nicconf->{link_down};
4934             }
4935         }
4936
4937         vm_mon_cmd_nocheck($vmid, 'qom-set',
4938                     path => "machine/peripheral/balloon0",
4939                     property => "guest-stats-polling-interval",
4940                     value => 2) if (!defined($conf->{balloon}) || $conf->{balloon});
4941
4942     });
4943 }
4944
4945 sub vm_mon_cmd {
4946     my ($vmid, $execute, %params) = @_;
4947
4948     my $cmd = { execute => $execute, arguments => \%params };
4949     vm_qmp_command($vmid, $cmd);
4950 }
4951
4952 sub vm_mon_cmd_nocheck {
4953     my ($vmid, $execute, %params) = @_;
4954
4955     my $cmd = { execute => $execute, arguments => \%params };
4956     vm_qmp_command($vmid, $cmd, 1);
4957 }
4958
4959 sub vm_qmp_command {
4960     my ($vmid, $cmd, $nocheck) = @_;
4961
4962     my $res;
4963
4964     my $timeout;
4965     if ($cmd->{arguments} && $cmd->{arguments}->{timeout}) {
4966         $timeout = $cmd->{arguments}->{timeout};
4967         delete $cmd->{arguments}->{timeout};
4968     }
4969
4970     eval {
4971         die "VM $vmid not running\n" if !check_running($vmid, $nocheck);
4972         my $sname = qmp_socket($vmid);
4973         if (-e $sname) { # test if VM is reasonambe new and supports qmp/qga
4974             my $qmpclient = PVE::QMPClient->new();
4975
4976             $res = $qmpclient->cmd($vmid, $cmd, $timeout);
4977         } else {
4978             die "unable to open monitor socket\n";
4979         }
4980     };
4981     if (my $err = $@) {
4982         syslog("err", "VM $vmid qmp command failed - $err");
4983         die $err;
4984     }
4985
4986     return $res;
4987 }
4988
4989 sub vm_human_monitor_command {
4990     my ($vmid, $cmdline) = @_;
4991
4992     my $res;
4993
4994     my $cmd = {
4995         execute => 'human-monitor-command',
4996         arguments => { 'command-line' => $cmdline},
4997     };
4998
4999     return vm_qmp_command($vmid, $cmd);
5000 }
5001
5002 sub vm_commandline {
5003     my ($storecfg, $vmid) = @_;
5004
5005     my $conf = PVE::QemuConfig->load_config($vmid);
5006
5007     my $defaults = load_defaults();
5008
5009     my $cmd = config_to_command($storecfg, $vmid, $conf, $defaults);
5010
5011     return PVE::Tools::cmd2string($cmd);
5012 }
5013
5014 sub vm_reset {
5015     my ($vmid, $skiplock) = @_;
5016
5017     PVE::QemuConfig->lock_config($vmid, sub {
5018
5019         my $conf = PVE::QemuConfig->load_config($vmid);
5020
5021         PVE::QemuConfig->check_lock($conf) if !$skiplock;
5022
5023         vm_mon_cmd($vmid, "system_reset");
5024     });
5025 }
5026
5027 sub get_vm_volumes {
5028     my ($conf) = @_;
5029
5030     my $vollist = [];
5031     foreach_volid($conf, sub {
5032         my ($volid, $attr) = @_;
5033
5034         return if $volid =~ m|^/|;
5035
5036         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
5037         return if !$sid;
5038
5039         push @$vollist, $volid;
5040     });
5041
5042     return $vollist;
5043 }
5044
5045 sub vm_stop_cleanup {
5046     my ($storecfg, $vmid, $conf, $keepActive, $apply_pending_changes) = @_;
5047
5048     eval {
5049
5050         if (!$keepActive) {
5051             my $vollist = get_vm_volumes($conf);
5052             PVE::Storage::deactivate_volumes($storecfg, $vollist);
5053         }
5054
5055         foreach my $ext (qw(mon qmp pid vnc qga)) {
5056             unlink "/var/run/qemu-server/${vmid}.$ext";
5057         }
5058
5059         vmconfig_apply_pending($vmid, $conf, $storecfg) if $apply_pending_changes;
5060     };
5061     warn $@ if $@; # avoid errors - just warn
5062 }
5063
5064 # Note: use $nockeck to skip tests if VM configuration file exists.
5065 # We need that when migration VMs to other nodes (files already moved)
5066 # Note: we set $keepActive in vzdump stop mode - volumes need to stay active
5067 sub vm_stop {
5068     my ($storecfg, $vmid, $skiplock, $nocheck, $timeout, $shutdown, $force, $keepActive, $migratedfrom) = @_;
5069
5070     $force = 1 if !defined($force) && !$shutdown;
5071
5072     if ($migratedfrom){
5073         my $pid = check_running($vmid, $nocheck, $migratedfrom);
5074         kill 15, $pid if $pid;
5075         my $conf = PVE::QemuConfig->load_config($vmid, $migratedfrom);
5076         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 0);
5077         return;
5078     }
5079
5080     PVE::QemuConfig->lock_config($vmid, sub {
5081
5082         my $pid = check_running($vmid, $nocheck);
5083         return if !$pid;
5084
5085         my $conf;
5086         if (!$nocheck) {
5087             $conf = PVE::QemuConfig->load_config($vmid);
5088             PVE::QemuConfig->check_lock($conf) if !$skiplock;
5089             if (!defined($timeout) && $shutdown && $conf->{startup}) {
5090                 my $opts = PVE::JSONSchema::pve_parse_startup_order($conf->{startup});
5091                 $timeout = $opts->{down} if $opts->{down};
5092             }
5093         }
5094
5095         $timeout = 60 if !defined($timeout);
5096
5097         eval {
5098             if ($shutdown) {
5099                 if (defined($conf) && $conf->{agent}) {
5100                     vm_qmp_command($vmid, { execute => "guest-shutdown" }, $nocheck);
5101                 } else {
5102                     vm_qmp_command($vmid, { execute => "system_powerdown" }, $nocheck);
5103                 }
5104             } else {
5105                 vm_qmp_command($vmid, { execute => "quit" }, $nocheck);
5106             }
5107         };
5108         my $err = $@;
5109
5110         if (!$err) {
5111             my $count = 0;
5112             while (($count < $timeout) && check_running($vmid, $nocheck)) {
5113                 $count++;
5114                 sleep 1;
5115             }
5116
5117             if ($count >= $timeout) {
5118                 if ($force) {
5119                     warn "VM still running - terminating now with SIGTERM\n";
5120                     kill 15, $pid;
5121                 } else {
5122                     die "VM quit/powerdown failed - got timeout\n";
5123                 }
5124             } else {
5125                 vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5126                 return;
5127             }
5128         } else {
5129             if ($force) {
5130                 warn "VM quit/powerdown failed - terminating now with SIGTERM\n";
5131                 kill 15, $pid;
5132             } else {
5133                 die "VM quit/powerdown failed\n";
5134             }
5135         }
5136
5137         # wait again
5138         $timeout = 10;
5139
5140         my $count = 0;
5141         while (($count < $timeout) && check_running($vmid, $nocheck)) {
5142             $count++;
5143             sleep 1;
5144         }
5145
5146         if ($count >= $timeout) {
5147             warn "VM still running - terminating now with SIGKILL\n";
5148             kill 9, $pid;
5149             sleep 1;
5150         }
5151
5152         vm_stop_cleanup($storecfg, $vmid, $conf, $keepActive, 1) if $conf;
5153    });
5154 }
5155
5156 sub vm_suspend {
5157     my ($vmid, $skiplock) = @_;
5158
5159     PVE::QemuConfig->lock_config($vmid, sub {
5160
5161         my $conf = PVE::QemuConfig->load_config($vmid);
5162
5163         PVE::QemuConfig->check_lock($conf)
5164             if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5165
5166         vm_mon_cmd($vmid, "stop");
5167     });
5168 }
5169
5170 sub vm_resume {
5171     my ($vmid, $skiplock, $nocheck) = @_;
5172
5173     PVE::QemuConfig->lock_config($vmid, sub {
5174
5175         if (!$nocheck) {
5176
5177             my $conf = PVE::QemuConfig->load_config($vmid);
5178
5179             PVE::QemuConfig->check_lock($conf)
5180                 if !($skiplock || PVE::QemuConfig->has_lock($conf, 'backup'));
5181
5182             vm_mon_cmd($vmid, "cont");
5183
5184         } else {
5185             vm_mon_cmd_nocheck($vmid, "cont");
5186         }
5187     });
5188 }
5189
5190 sub vm_sendkey {
5191     my ($vmid, $skiplock, $key) = @_;
5192
5193     PVE::QemuConfig->lock_config($vmid, sub {
5194
5195         my $conf = PVE::QemuConfig->load_config($vmid);
5196
5197         # there is no qmp command, so we use the human monitor command
5198         vm_human_monitor_command($vmid, "sendkey $key");
5199     });
5200 }
5201
5202 sub vm_destroy {
5203     my ($storecfg, $vmid, $skiplock) = @_;
5204
5205     PVE::QemuConfig->lock_config($vmid, sub {
5206
5207         my $conf = PVE::QemuConfig->load_config($vmid);
5208
5209         if (!check_running($vmid)) {
5210             destroy_vm($storecfg, $vmid, undef, $skiplock);
5211         } else {
5212             die "VM $vmid is running - destroy failed\n";
5213         }
5214     });
5215 }
5216
5217 # pci helpers
5218
5219 sub file_write {
5220     my ($filename, $buf) = @_;
5221
5222     my $fh = IO::File->new($filename, "w");
5223     return undef if !$fh;
5224
5225     my $res = print $fh $buf;
5226
5227     $fh->close();
5228
5229     return $res;
5230 }
5231
5232 sub pci_device_info {
5233     my ($name) = @_;
5234
5235     my $res;
5236
5237     return undef if $name !~ m/^([a-f0-9]{4}):([a-f0-9]{2}):([a-f0-9]{2})\.([a-f0-9])$/;
5238     my ($domain, $bus, $slot, $func) = ($1, $2, $3, $4);
5239
5240     my $irq = file_read_firstline("$pcisysfs/devices/$name/irq");
5241     return undef if !defined($irq) || $irq !~ m/^\d+$/;
5242
5243     my $vendor = file_read_firstline("$pcisysfs/devices/$name/vendor");
5244     return undef if !defined($vendor) || $vendor !~ s/^0x//;
5245
5246     my $product = file_read_firstline("$pcisysfs/devices/$name/device");
5247     return undef if !defined($product) || $product !~ s/^0x//;
5248
5249     $res = {
5250         name => $name,
5251         vendor => $vendor,
5252         product => $product,
5253         domain => $domain,
5254         bus => $bus,
5255         slot => $slot,
5256         func => $func,
5257         irq => $irq,
5258         has_fl_reset => -f "$pcisysfs/devices/$name/reset" || 0,
5259     };
5260
5261     return $res;
5262 }
5263
5264 sub pci_dev_reset {
5265     my ($dev) = @_;
5266
5267     my $name = $dev->{name};
5268
5269     my $fn = "$pcisysfs/devices/$name/reset";
5270
5271     return file_write($fn, "1");
5272 }
5273
5274 sub pci_dev_bind_to_vfio {
5275     my ($dev) = @_;
5276
5277     my $name = $dev->{name};
5278
5279     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5280
5281     if (!-d $vfio_basedir) {
5282         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5283     }
5284     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5285
5286     my $testdir = "$vfio_basedir/$name";
5287     return 1 if -d $testdir;
5288
5289     my $data = "$dev->{vendor} $dev->{product}";
5290     return undef if !file_write("$vfio_basedir/new_id", $data);
5291
5292     my $fn = "$pcisysfs/devices/$name/driver/unbind";
5293     if (!file_write($fn, $name)) {
5294         return undef if -f $fn;
5295     }
5296
5297     $fn = "$vfio_basedir/bind";
5298     if (! -d $testdir) {
5299         return undef if !file_write($fn, $name);
5300     }
5301
5302     return -d $testdir;
5303 }
5304
5305 sub pci_dev_group_bind_to_vfio {
5306     my ($pciid) = @_;
5307
5308     my $vfio_basedir = "$pcisysfs/drivers/vfio-pci";
5309
5310     if (!-d $vfio_basedir) {
5311         system("/sbin/modprobe vfio-pci >/dev/null 2>/dev/null");
5312     }
5313     die "Cannot find vfio-pci module!\n" if !-d $vfio_basedir;
5314
5315     # get IOMMU group devices
5316     opendir(my $D, "$pcisysfs/devices/0000:$pciid/iommu_group/devices/") || die "Cannot open iommu_group: $!\n";
5317       my @devs = grep /^0000:/, readdir($D);
5318     closedir($D);
5319
5320     foreach my $pciid (@devs) {
5321         $pciid =~ m/^([:\.\da-f]+)$/ or die "PCI ID $pciid not valid!\n";
5322
5323         # pci bridges, switches or root ports are not supported
5324         # they have a pci_bus subdirectory so skip them
5325         next if (-e "$pcisysfs/devices/$pciid/pci_bus");
5326
5327         my $info = pci_device_info($1);
5328         pci_dev_bind_to_vfio($info) || die "Cannot bind $pciid to vfio\n";
5329     }
5330
5331     return 1;
5332 }
5333
5334 # vzdump restore implementaion
5335
5336 sub tar_archive_read_firstfile {
5337     my $archive = shift;
5338
5339     die "ERROR: file '$archive' does not exist\n" if ! -f $archive;
5340
5341     # try to detect archive type first
5342     my $pid = open (my $fh, '-|', 'tar', 'tf', $archive) ||
5343         die "unable to open file '$archive'\n";
5344     my $firstfile = <$fh>;
5345     kill 15, $pid;
5346     close $fh;
5347
5348     die "ERROR: archive contaions no data\n" if !$firstfile;
5349     chomp $firstfile;
5350
5351     return $firstfile;
5352 }
5353
5354 sub tar_restore_cleanup {
5355     my ($storecfg, $statfile) = @_;
5356
5357     print STDERR "starting cleanup\n";
5358
5359     if (my $fd = IO::File->new($statfile, "r")) {
5360         while (defined(my $line = <$fd>)) {
5361             if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5362                 my $volid = $2;
5363                 eval {
5364                     if ($volid =~ m|^/|) {
5365                         unlink $volid || die 'unlink failed\n';
5366                     } else {
5367                         PVE::Storage::vdisk_free($storecfg, $volid);
5368                     }
5369                     print STDERR "temporary volume '$volid' sucessfuly removed\n";
5370                 };
5371                 print STDERR "unable to cleanup '$volid' - $@" if $@;
5372             } else {
5373                 print STDERR "unable to parse line in statfile - $line";
5374             }
5375         }
5376         $fd->close();
5377     }
5378 }
5379
5380 sub restore_archive {
5381     my ($archive, $vmid, $user, $opts) = @_;
5382
5383     my $format = $opts->{format};
5384     my $comp;
5385
5386     if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
5387         $format = 'tar' if !$format;
5388         $comp = 'gzip';
5389     } elsif ($archive =~ m/\.tar$/) {
5390         $format = 'tar' if !$format;
5391     } elsif ($archive =~ m/.tar.lzo$/) {
5392         $format = 'tar' if !$format;
5393         $comp = 'lzop';
5394     } elsif ($archive =~ m/\.vma$/) {
5395         $format = 'vma' if !$format;
5396     } elsif ($archive =~ m/\.vma\.gz$/) {
5397         $format = 'vma' if !$format;
5398         $comp = 'gzip';
5399     } elsif ($archive =~ m/\.vma\.lzo$/) {
5400         $format = 'vma' if !$format;
5401         $comp = 'lzop';
5402     } else {
5403         $format = 'vma' if !$format; # default
5404     }
5405
5406     # try to detect archive format
5407     if ($format eq 'tar') {
5408         return restore_tar_archive($archive, $vmid, $user, $opts);
5409     } else {
5410         return restore_vma_archive($archive, $vmid, $user, $opts, $comp);
5411     }
5412 }
5413
5414 sub restore_update_config_line {
5415     my ($outfd, $cookie, $vmid, $map, $line, $unique) = @_;
5416
5417     return if $line =~ m/^\#qmdump\#/;
5418     return if $line =~ m/^\#vzdump\#/;
5419     return if $line =~ m/^lock:/;
5420     return if $line =~ m/^unused\d+:/;
5421     return if $line =~ m/^parent:/;
5422     return if $line =~ m/^template:/; # restored VM is never a template
5423
5424     my $dc = PVE::Cluster::cfs_read_file('datacenter.cfg');
5425     if (($line =~ m/^(vlan(\d+)):\s*(\S+)\s*$/)) {
5426         # try to convert old 1.X settings
5427         my ($id, $ind, $ethcfg) = ($1, $2, $3);
5428         foreach my $devconfig (PVE::Tools::split_list($ethcfg)) {
5429             my ($model, $macaddr) = split(/\=/, $devconfig);
5430             $macaddr = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if !$macaddr || $unique;
5431             my $net = {
5432                 model => $model,
5433                 bridge => "vmbr$ind",
5434                 macaddr => $macaddr,
5435             };
5436             my $netstr = print_net($net);
5437
5438             print $outfd "net$cookie->{netcount}: $netstr\n";
5439             $cookie->{netcount}++;
5440         }
5441     } elsif (($line =~ m/^(net\d+):\s*(\S+)\s*$/) && $unique) {
5442         my ($id, $netstr) = ($1, $2);
5443         my $net = parse_net($netstr);
5444         $net->{macaddr} = PVE::Tools::random_ether_addr($dc->{mac_prefix}) if $net->{macaddr};
5445         $netstr = print_net($net);
5446         print $outfd "$id: $netstr\n";
5447     } elsif ($line =~ m/^((ide|scsi|virtio|sata|efidisk)\d+):\s*(\S+)\s*$/) {
5448         my $virtdev = $1;
5449         my $value = $3;
5450         my $di = parse_drive($virtdev, $value);
5451         if (defined($di->{backup}) && !$di->{backup}) {
5452             print $outfd "#$line";
5453         } elsif ($map->{$virtdev}) {
5454             delete $di->{format}; # format can change on restore
5455             $di->{file} = $map->{$virtdev};
5456             $value = print_drive($vmid, $di);
5457             print $outfd "$virtdev: $value\n";
5458         } else {
5459             print $outfd $line;
5460         }
5461     } elsif (($line =~ m/^(smbios1: )(.*)/) && $unique) {
5462         my ($uuid, $uuid_str);
5463         UUID::generate($uuid);
5464         UUID::unparse($uuid, $uuid_str);
5465         my $smbios1 = parse_smbios1($2);
5466         $smbios1->{uuid} = $uuid_str;
5467         print $outfd $1.print_smbios1($smbios1)."\n";
5468     } else {
5469         print $outfd $line;
5470     }
5471 }
5472
5473 sub scan_volids {
5474     my ($cfg, $vmid) = @_;
5475
5476     my $info = PVE::Storage::vdisk_list($cfg, undef, $vmid);
5477
5478     my $volid_hash = {};
5479     foreach my $storeid (keys %$info) {
5480         foreach my $item (@{$info->{$storeid}}) {
5481             next if !($item->{volid} && $item->{size});
5482             $item->{path} = PVE::Storage::path($cfg, $item->{volid});
5483             $volid_hash->{$item->{volid}} = $item;
5484         }
5485     }
5486
5487     return $volid_hash;
5488 }
5489
5490 sub is_volume_in_use {
5491     my ($storecfg, $conf, $skip_drive, $volid) = @_;
5492
5493     my $path = PVE::Storage::path($storecfg, $volid);
5494
5495     my $scan_config = sub {
5496         my ($cref, $snapname) = @_;
5497
5498         foreach my $key (keys %$cref) {
5499             my $value = $cref->{$key};
5500             if (is_valid_drivename($key)) {
5501                 next if $skip_drive && $key eq $skip_drive;
5502                 my $drive = parse_drive($key, $value);
5503                 next if !$drive || !$drive->{file} || drive_is_cdrom($drive);
5504                 return 1 if $volid eq $drive->{file};
5505                 if ($drive->{file} =~ m!^/!) {
5506                     return 1 if $drive->{file} eq $path;
5507                 } else {
5508                     my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file}, 1);
5509                     next if !$storeid;
5510                     my $scfg = PVE::Storage::storage_config($storecfg, $storeid, 1);
5511                     next if !$scfg;
5512                     return 1 if $path eq PVE::Storage::path($storecfg, $drive->{file}, $snapname);
5513                 }
5514             }
5515         }
5516
5517         return 0;
5518     };
5519
5520     return 1 if &$scan_config($conf);
5521
5522     undef $skip_drive;
5523
5524     foreach my $snapname (keys %{$conf->{snapshots}}) {
5525         return 1 if &$scan_config($conf->{snapshots}->{$snapname}, $snapname);
5526     }
5527
5528     return 0;
5529 }
5530
5531 sub update_disksize {
5532     my ($vmid, $conf, $volid_hash) = @_;
5533
5534     my $changes;
5535
5536     # used and unused disks
5537     my $referenced = {};
5538
5539     # Note: it is allowed to define multiple storages with same path (alias), so
5540     # we need to check both 'volid' and real 'path' (two different volid can point
5541     # to the same path).
5542
5543     my $referencedpath = {};
5544
5545     # update size info
5546     foreach my $opt (keys %$conf) {
5547         if (is_valid_drivename($opt)) {
5548             my $drive = parse_drive($opt, $conf->{$opt});
5549             my $volid = $drive->{file};
5550             next if !$volid;
5551
5552             $referenced->{$volid} = 1;
5553             if ($volid_hash->{$volid} &&
5554                 (my $path = $volid_hash->{$volid}->{path})) {
5555                 $referencedpath->{$path} = 1;
5556             }
5557
5558             next if drive_is_cdrom($drive);
5559             next if !$volid_hash->{$volid};
5560
5561             $drive->{size} = $volid_hash->{$volid}->{size};
5562             my $new = print_drive($vmid, $drive);
5563             if ($new ne $conf->{$opt}) {
5564                 $changes = 1;
5565                 $conf->{$opt} = $new;
5566             }
5567         }
5568     }
5569
5570     # remove 'unusedX' entry if volume is used
5571     foreach my $opt (keys %$conf) {
5572         next if $opt !~ m/^unused\d+$/;
5573         my $volid = $conf->{$opt};
5574         my $path = $volid_hash->{$volid}->{path} if $volid_hash->{$volid};
5575         if ($referenced->{$volid} || ($path && $referencedpath->{$path})) {
5576             $changes = 1;
5577             delete $conf->{$opt};
5578         }
5579
5580         $referenced->{$volid} = 1;
5581         $referencedpath->{$path} = 1 if $path;
5582     }
5583
5584     foreach my $volid (sort keys %$volid_hash) {
5585         next if $volid =~ m/vm-$vmid-state-/;
5586         next if $referenced->{$volid};
5587         my $path = $volid_hash->{$volid}->{path};
5588         next if !$path; # just to be sure
5589         next if $referencedpath->{$path};
5590         $changes = 1;
5591         PVE::QemuConfig->add_unused_volume($conf, $volid);
5592         $referencedpath->{$path} = 1; # avoid to add more than once (aliases)
5593     }
5594
5595     return $changes;
5596 }
5597
5598 sub rescan {
5599     my ($vmid, $nolock) = @_;
5600
5601     my $cfg = PVE::Storage::config();
5602
5603     my $volid_hash = scan_volids($cfg, $vmid);
5604
5605     my $updatefn =  sub {
5606         my ($vmid) = @_;
5607
5608         my $conf = PVE::QemuConfig->load_config($vmid);
5609
5610         PVE::QemuConfig->check_lock($conf);
5611
5612         my $vm_volids = {};
5613         foreach my $volid (keys %$volid_hash) {
5614             my $info = $volid_hash->{$volid};
5615             $vm_volids->{$volid} = $info if $info->{vmid} && $info->{vmid} == $vmid;
5616         }
5617
5618         my $changes = update_disksize($vmid, $conf, $vm_volids);
5619
5620         PVE::QemuConfig->write_config($vmid, $conf) if $changes;
5621     };
5622
5623     if (defined($vmid)) {
5624         if ($nolock) {
5625             &$updatefn($vmid);
5626         } else {
5627             PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5628         }
5629     } else {
5630         my $vmlist = config_list();
5631         foreach my $vmid (keys %$vmlist) {
5632             if ($nolock) {
5633                 &$updatefn($vmid);
5634             } else {
5635                 PVE::QemuConfig->lock_config($vmid, $updatefn, $vmid);
5636             }
5637         }
5638     }
5639 }
5640
5641 sub restore_vma_archive {
5642     my ($archive, $vmid, $user, $opts, $comp) = @_;
5643
5644     my $input = $archive eq '-' ? "<&STDIN" : undef;
5645     my $readfrom = $archive;
5646
5647     my $uncomp = '';
5648     if ($comp) {
5649         $readfrom = '-';
5650         my $qarchive = PVE::Tools::shellquote($archive);
5651         if ($comp eq 'gzip') {
5652             $uncomp = "zcat $qarchive|";
5653         } elsif ($comp eq 'lzop') {
5654             $uncomp = "lzop -d -c $qarchive|";
5655         } else {
5656             die "unknown compression method '$comp'\n";
5657         }
5658
5659     }
5660
5661     my $tmpdir = "/var/tmp/vzdumptmp$$";
5662     rmtree $tmpdir;
5663
5664     # disable interrupts (always do cleanups)
5665     local $SIG{INT} =
5666         local $SIG{TERM} =
5667         local $SIG{QUIT} =
5668         local $SIG{HUP} = sub { warn "got interrupt - ignored\n"; };
5669
5670     my $mapfifo = "/var/tmp/vzdumptmp$$.fifo";
5671     POSIX::mkfifo($mapfifo, 0600);
5672     my $fifofh;
5673
5674     my $openfifo = sub {
5675         open($fifofh, '>', $mapfifo) || die $!;
5676     };
5677
5678     my $cmd = "${uncomp}vma extract -v -r $mapfifo $readfrom $tmpdir";
5679
5680     my $oldtimeout;
5681     my $timeout = 5;
5682
5683     my $devinfo = {};
5684
5685     my $rpcenv = PVE::RPCEnvironment::get();
5686
5687     my $conffile = PVE::QemuConfig->config_file($vmid);
5688     my $tmpfn = "$conffile.$$.tmp";
5689
5690     # Note: $oldconf is undef if VM does not exists
5691     my $cfs_path = PVE::QemuConfig->cfs_config_path($vmid);
5692     my $oldconf = PVE::Cluster::cfs_read_file($cfs_path);
5693
5694     my $print_devmap = sub {
5695         my $virtdev_hash = {};
5696
5697         my $cfgfn = "$tmpdir/qemu-server.conf";
5698
5699         # we can read the config - that is already extracted
5700         my $fh = IO::File->new($cfgfn, "r") ||
5701             "unable to read qemu-server.conf - $!\n";
5702
5703         my $fwcfgfn = "$tmpdir/qemu-server.fw";
5704         if (-f $fwcfgfn) {
5705             my $pve_firewall_dir = '/etc/pve/firewall';
5706             mkdir $pve_firewall_dir; # make sure the dir exists
5707             PVE::Tools::file_copy($fwcfgfn, "${pve_firewall_dir}/$vmid.fw");
5708         }
5709
5710         while (defined(my $line = <$fh>)) {
5711             if ($line =~ m/^\#qmdump\#map:(\S+):(\S+):(\S*):(\S*):$/) {
5712                 my ($virtdev, $devname, $storeid, $format) = ($1, $2, $3, $4);
5713                 die "archive does not contain data for drive '$virtdev'\n"
5714                     if !$devinfo->{$devname};
5715                 if (defined($opts->{storage})) {
5716                     $storeid = $opts->{storage} || 'local';
5717                 } elsif (!$storeid) {
5718                     $storeid = 'local';
5719                 }
5720                 $format = 'raw' if !$format;
5721                 $devinfo->{$devname}->{devname} = $devname;
5722                 $devinfo->{$devname}->{virtdev} = $virtdev;
5723                 $devinfo->{$devname}->{format} = $format;
5724                 $devinfo->{$devname}->{storeid} = $storeid;
5725
5726                 # check permission on storage
5727                 my $pool = $opts->{pool}; # todo: do we need that?
5728                 if ($user ne 'root@pam') {
5729                     $rpcenv->check($user, "/storage/$storeid", ['Datastore.AllocateSpace']);
5730                 }
5731
5732                 $virtdev_hash->{$virtdev} = $devinfo->{$devname};
5733             }
5734         }
5735
5736         foreach my $devname (keys %$devinfo) {
5737             die "found no device mapping information for device '$devname'\n"
5738                 if !$devinfo->{$devname}->{virtdev};
5739         }
5740
5741         my $cfg = PVE::Storage::config();
5742
5743         # create empty/temp config
5744         if ($oldconf) {
5745             PVE::Tools::file_set_contents($conffile, "memory: 128\n");
5746             foreach_drive($oldconf, sub {
5747                 my ($ds, $drive) = @_;
5748
5749                 return if drive_is_cdrom($drive);
5750
5751                 my $volid = $drive->{file};
5752
5753                 return if !$volid || $volid =~ m|^/|;
5754
5755                 my ($path, $owner) = PVE::Storage::path($cfg, $volid);
5756                 return if !$path || !$owner || ($owner != $vmid);
5757
5758                 # Note: only delete disk we want to restore
5759                 # other volumes will become unused
5760                 if ($virtdev_hash->{$ds}) {
5761                     eval { PVE::Storage::vdisk_free($cfg, $volid); };
5762                     if (my $err = $@) {
5763                         warn $err;
5764                     }
5765                 }
5766             });
5767
5768             # delete vmstate files
5769             # since after the restore we have no snapshots anymore
5770             foreach my $snapname (keys %{$oldconf->{snapshots}}) {
5771                 my $snap = $oldconf->{snapshots}->{$snapname};
5772                 if ($snap->{vmstate}) {
5773                     eval { PVE::Storage::vdisk_free($cfg, $snap->{vmstate}); };
5774                     if (my $err = $@) {
5775                         warn $err;
5776                     }
5777                 }
5778             }
5779         }
5780
5781         my $map = {};
5782         foreach my $virtdev (sort keys %$virtdev_hash) {
5783             my $d = $virtdev_hash->{$virtdev};
5784             my $alloc_size = int(($d->{size} + 1024 - 1)/1024);
5785             my $scfg = PVE::Storage::storage_config($cfg, $d->{storeid});
5786
5787             # test if requested format is supported
5788             my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($cfg, $d->{storeid});
5789             my $supported = grep { $_ eq $d->{format} } @$validFormats;
5790             $d->{format} = $defFormat if !$supported;
5791
5792             my $volid = PVE::Storage::vdisk_alloc($cfg, $d->{storeid}, $vmid,
5793                                                   $d->{format}, undef, $alloc_size);
5794             print STDERR "new volume ID is '$volid'\n";
5795             $d->{volid} = $volid;
5796             my $path = PVE::Storage::path($cfg, $volid);
5797
5798             PVE::Storage::activate_volumes($cfg,[$volid]);
5799
5800             my $write_zeros = 1;
5801             if (PVE::Storage::volume_has_feature($cfg, 'sparseinit', $volid)) {
5802                 $write_zeros = 0;
5803             }
5804
5805             print $fifofh "format=$d->{format}:${write_zeros}:$d->{devname}=$path\n";
5806
5807             print "map '$d->{devname}' to '$path' (write zeros = ${write_zeros})\n";
5808             $map->{$virtdev} = $volid;
5809         }
5810
5811         $fh->seek(0, 0) || die "seek failed - $!\n";
5812
5813         my $outfd = new IO::File ($tmpfn, "w") ||
5814             die "unable to write config for VM $vmid\n";
5815
5816         my $cookie = { netcount => 0 };
5817         while (defined(my $line = <$fh>)) {
5818             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5819         }
5820
5821         $fh->close();
5822         $outfd->close();
5823     };
5824
5825     eval {
5826         # enable interrupts
5827         local $SIG{INT} =
5828             local $SIG{TERM} =
5829             local $SIG{QUIT} =
5830             local $SIG{HUP} =
5831             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5832         local $SIG{ALRM} = sub { die "got timeout\n"; };
5833
5834         $oldtimeout = alarm($timeout);
5835
5836         my $parser = sub {
5837             my $line = shift;
5838
5839             print "$line\n";
5840
5841             if ($line =~ m/^DEV:\sdev_id=(\d+)\ssize:\s(\d+)\sdevname:\s(\S+)$/) {
5842                 my ($dev_id, $size, $devname) = ($1, $2, $3);
5843                 $devinfo->{$devname} = { size => $size, dev_id => $dev_id };
5844             } elsif ($line =~ m/^CTIME: /) {
5845                 # we correctly received the vma config, so we can disable
5846                 # the timeout now for disk allocation (set to 10 minutes, so
5847                 # that we always timeout if something goes wrong)
5848                 alarm(600);
5849                 &$print_devmap();
5850                 print $fifofh "done\n";
5851                 my $tmp = $oldtimeout || 0;
5852                 $oldtimeout = undef;
5853                 alarm($tmp);
5854                 close($fifofh);
5855             }
5856         };
5857
5858         print "restore vma archive: $cmd\n";
5859         run_command($cmd, input => $input, outfunc => $parser, afterfork => $openfifo);
5860     };
5861     my $err = $@;
5862
5863     alarm($oldtimeout) if $oldtimeout;
5864
5865     my $vollist = [];
5866     foreach my $devname (keys %$devinfo) {
5867         my $volid = $devinfo->{$devname}->{volid};
5868         push @$vollist, $volid if $volid;
5869     }
5870
5871     my $cfg = PVE::Storage::config();
5872     PVE::Storage::deactivate_volumes($cfg, $vollist);
5873
5874     unlink $mapfifo;
5875
5876     if ($err) {
5877         rmtree $tmpdir;
5878         unlink $tmpfn;
5879
5880         foreach my $devname (keys %$devinfo) {
5881             my $volid = $devinfo->{$devname}->{volid};
5882             next if !$volid;
5883             eval {
5884                 if ($volid =~ m|^/|) {
5885                     unlink $volid || die 'unlink failed\n';
5886                 } else {
5887                     PVE::Storage::vdisk_free($cfg, $volid);
5888                 }
5889                 print STDERR "temporary volume '$volid' sucessfuly removed\n";
5890             };
5891             print STDERR "unable to cleanup '$volid' - $@" if $@;
5892         }
5893         die $err;
5894     }
5895
5896     rmtree $tmpdir;
5897
5898     rename($tmpfn, $conffile) ||
5899         die "unable to commit configuration file '$conffile'\n";
5900
5901     PVE::Cluster::cfs_update(); # make sure we read new file
5902
5903     eval { rescan($vmid, 1); };
5904     warn $@ if $@;
5905 }
5906
5907 sub restore_tar_archive {
5908     my ($archive, $vmid, $user, $opts) = @_;
5909
5910     if ($archive ne '-') {
5911         my $firstfile = tar_archive_read_firstfile($archive);
5912         die "ERROR: file '$archive' dos not lock like a QemuServer vzdump backup\n"
5913             if $firstfile ne 'qemu-server.conf';
5914     }
5915
5916     my $storecfg = PVE::Storage::config();
5917
5918     # destroy existing data - keep empty config
5919     my $vmcfgfn = PVE::QemuConfig->config_file($vmid);
5920     destroy_vm($storecfg, $vmid, 1) if -f $vmcfgfn;
5921
5922     my $tocmd = "/usr/lib/qemu-server/qmextract";
5923
5924     $tocmd .= " --storage " . PVE::Tools::shellquote($opts->{storage}) if $opts->{storage};
5925     $tocmd .= " --pool " . PVE::Tools::shellquote($opts->{pool}) if $opts->{pool};
5926     $tocmd .= ' --prealloc' if $opts->{prealloc};
5927     $tocmd .= ' --info' if $opts->{info};
5928
5929     # tar option "xf" does not autodetect compression when read from STDIN,
5930     # so we pipe to zcat
5931     my $cmd = "zcat -f|tar xf " . PVE::Tools::shellquote($archive) . " " .
5932         PVE::Tools::shellquote("--to-command=$tocmd");
5933
5934     my $tmpdir = "/var/tmp/vzdumptmp$$";
5935     mkpath $tmpdir;
5936
5937     local $ENV{VZDUMP_TMPDIR} = $tmpdir;
5938     local $ENV{VZDUMP_VMID} = $vmid;
5939     local $ENV{VZDUMP_USER} = $user;
5940
5941     my $conffile = PVE::QemuConfig->config_file($vmid);
5942     my $tmpfn = "$conffile.$$.tmp";
5943
5944     # disable interrupts (always do cleanups)
5945     local $SIG{INT} =
5946         local $SIG{TERM} =
5947         local $SIG{QUIT} =
5948         local $SIG{HUP} = sub { print STDERR "got interrupt - ignored\n"; };
5949
5950     eval {
5951         # enable interrupts
5952         local $SIG{INT} =
5953             local $SIG{TERM} =
5954             local $SIG{QUIT} =
5955             local $SIG{HUP} =
5956             local $SIG{PIPE} = sub { die "interrupted by signal\n"; };
5957
5958         if ($archive eq '-') {
5959             print "extracting archive from STDIN\n";
5960             run_command($cmd, input => "<&STDIN");
5961         } else {
5962             print "extracting archive '$archive'\n";
5963             run_command($cmd);
5964         }
5965
5966         return if $opts->{info};
5967
5968         # read new mapping
5969         my $map = {};
5970         my $statfile = "$tmpdir/qmrestore.stat";
5971         if (my $fd = IO::File->new($statfile, "r")) {
5972             while (defined (my $line = <$fd>)) {
5973                 if ($line =~ m/vzdump:([^\s:]*):(\S+)$/) {
5974                     $map->{$1} = $2 if $1;
5975                 } else {
5976                     print STDERR "unable to parse line in statfile - $line\n";
5977                 }
5978             }
5979             $fd->close();
5980         }
5981
5982         my $confsrc = "$tmpdir/qemu-server.conf";
5983
5984         my $srcfd = new IO::File($confsrc, "r") ||
5985             die "unable to open file '$confsrc'\n";
5986
5987         my $outfd = new IO::File ($tmpfn, "w") ||
5988             die "unable to write config for VM $vmid\n";
5989
5990         my $cookie = { netcount => 0 };
5991         while (defined (my $line = <$srcfd>)) {
5992             restore_update_config_line($outfd, $cookie, $vmid, $map, $line, $opts->{unique});
5993         }
5994
5995         $srcfd->close();
5996         $outfd->close();
5997     };
5998     my $err = $@;
5999
6000     if ($err) {
6001
6002         unlink $tmpfn;
6003
6004         tar_restore_cleanup($storecfg, "$tmpdir/qmrestore.stat") if !$opts->{info};
6005
6006         die $err;
6007     }
6008
6009     rmtree $tmpdir;
6010
6011     rename $tmpfn, $conffile ||
6012         die "unable to commit configuration file '$conffile'\n";
6013
6014     PVE::Cluster::cfs_update(); # make sure we read new file
6015
6016     eval { rescan($vmid, 1); };
6017     warn $@ if $@;
6018 };
6019
6020 sub foreach_storage_used_by_vm {
6021     my ($conf, $func) = @_;
6022
6023     my $sidhash = {};
6024
6025     foreach_drive($conf, sub {
6026         my ($ds, $drive) = @_;
6027         return if drive_is_cdrom($drive);
6028
6029         my $volid = $drive->{file};
6030
6031         my ($sid, $volname) = PVE::Storage::parse_volume_id($volid, 1);
6032         $sidhash->{$sid} = $sid if $sid;
6033     });
6034
6035     foreach my $sid (sort keys %$sidhash) {
6036         &$func($sid);
6037     }
6038 }
6039
6040 sub do_snapshots_with_qemu {
6041     my ($storecfg, $volid) = @_;
6042
6043     my $storage_name = PVE::Storage::parse_volume_id($volid);
6044
6045     if ($qemu_snap_storage->{$storecfg->{ids}->{$storage_name}->{type}}
6046         && !$storecfg->{ids}->{$storage_name}->{krbd}){
6047         return 1;
6048     }
6049
6050     if ($volid =~ m/\.(qcow2|qed)$/){
6051         return 1;
6052     }
6053
6054     return undef;
6055 }
6056
6057 sub qga_check_running {
6058     my ($vmid) = @_;
6059
6060     eval { vm_mon_cmd($vmid, "guest-ping", timeout => 3); };
6061     if ($@) {
6062         warn "Qemu Guest Agent is not running - $@";
6063         return 0;
6064     }
6065     return 1;
6066 }
6067
6068 sub template_create {
6069     my ($vmid, $conf, $disk) = @_;
6070
6071     my $storecfg = PVE::Storage::config();
6072
6073     foreach_drive($conf, sub {
6074         my ($ds, $drive) = @_;
6075
6076         return if drive_is_cdrom($drive);
6077         return if $disk && $ds ne $disk;
6078
6079         my $volid = $drive->{file};
6080         return if !PVE::Storage::volume_has_feature($storecfg, 'template', $volid);
6081
6082         my $voliddst = PVE::Storage::vdisk_create_base($storecfg, $volid);
6083         $drive->{file} = $voliddst;
6084         $conf->{$ds} = print_drive($vmid, $drive);
6085         PVE::QemuConfig->write_config($vmid, $conf);
6086     });
6087 }
6088
6089 sub qemu_img_convert {
6090     my ($src_volid, $dst_volid, $size, $snapname, $is_zero_initialized) = @_;
6091
6092     my $storecfg = PVE::Storage::config();
6093     my ($src_storeid, $src_volname) = PVE::Storage::parse_volume_id($src_volid, 1);
6094     my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid, 1);
6095
6096     if ($src_storeid && $dst_storeid) {
6097
6098         PVE::Storage::activate_volumes($storecfg, [$src_volid], $snapname);
6099
6100         my $src_scfg = PVE::Storage::storage_config($storecfg, $src_storeid);
6101         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6102
6103         my $src_format = qemu_img_format($src_scfg, $src_volname);
6104         my $dst_format = qemu_img_format($dst_scfg, $dst_volname);
6105
6106         my $src_path = PVE::Storage::path($storecfg, $src_volid, $snapname);
6107         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6108
6109         my $cmd = [];
6110         push @$cmd, '/usr/bin/qemu-img', 'convert', '-p', '-n';
6111         push @$cmd, '-s', $snapname if($snapname && $src_format eq "qcow2");
6112         push @$cmd, '-f', $src_format, '-O', $dst_format, $src_path;
6113         if ($is_zero_initialized) {
6114             push @$cmd, "zeroinit:$dst_path";
6115         } else {
6116             push @$cmd, $dst_path;
6117         }
6118
6119         my $parser = sub {
6120             my $line = shift;
6121             if($line =~ m/\((\S+)\/100\%\)/){
6122                 my $percent = $1;
6123                 my $transferred = int($size * $percent / 100);
6124                 my $remaining = $size - $transferred;
6125
6126                 print "transferred: $transferred bytes remaining: $remaining bytes total: $size bytes progression: $percent %\n";
6127             }
6128
6129         };
6130
6131         eval  { run_command($cmd, timeout => undef, outfunc => $parser); };
6132         my $err = $@;
6133         die "copy failed: $err" if $err;
6134     }
6135 }
6136
6137 sub qemu_img_format {
6138     my ($scfg, $volname) = @_;
6139
6140     if ($scfg->{path} && $volname =~ m/\.($QEMU_FORMAT_RE)$/) {
6141         return $1;
6142     } else {
6143         return "raw";
6144     }
6145 }
6146
6147 sub qemu_drive_mirror {
6148     my ($vmid, $drive, $dst_volid, $vmiddst, $is_zero_initialized, $jobs, $skipcomplete, $qga) = @_;
6149
6150     $jobs = {} if !$jobs;
6151
6152     my $qemu_target;
6153     my $format;
6154     $jobs->{"drive-$drive"} = {};
6155
6156     if ($dst_volid =~ /^nbd:/) {
6157         $qemu_target = $dst_volid;
6158         $format = "nbd";
6159     } else {
6160         my $storecfg = PVE::Storage::config();
6161         my ($dst_storeid, $dst_volname) = PVE::Storage::parse_volume_id($dst_volid);
6162
6163         my $dst_scfg = PVE::Storage::storage_config($storecfg, $dst_storeid);
6164
6165         $format = qemu_img_format($dst_scfg, $dst_volname);
6166
6167         my $dst_path = PVE::Storage::path($storecfg, $dst_volid);
6168
6169         $qemu_target = $is_zero_initialized ? "zeroinit:$dst_path" : $dst_path;
6170     }
6171
6172     my $opts = { timeout => 10, device => "drive-$drive", mode => "existing", sync => "full", target => $qemu_target };
6173     $opts->{format} = $format if $format;
6174
6175     print "drive mirror is starting for drive-$drive\n";
6176
6177     eval { vm_mon_cmd($vmid, "drive-mirror", %$opts); }; #if a job already run for this device,it's throw an error
6178
6179     if (my $err = $@) {
6180         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6181         die "mirroring error: $err";
6182     }
6183
6184     qemu_drive_mirror_monitor ($vmid, $vmiddst, $jobs, $skipcomplete, $qga);
6185 }
6186
6187 sub qemu_drive_mirror_monitor {
6188     my ($vmid, $vmiddst, $jobs, $skipcomplete, $qga) = @_;
6189
6190     eval {
6191         my $err_complete = 0;
6192
6193         while (1) {
6194             die "storage migration timed out\n" if $err_complete > 300;
6195
6196             my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6197
6198             my $running_mirror_jobs = {};
6199             foreach my $stat (@$stats) {
6200                 next if $stat->{type} ne 'mirror';
6201                 $running_mirror_jobs->{$stat->{device}} = $stat;
6202             }
6203
6204             my $readycounter = 0;
6205
6206             foreach my $job (keys %$jobs) {
6207
6208                 if(defined($jobs->{$job}->{complete}) && !defined($running_mirror_jobs->{$job})) {
6209                     print "$job : finished\n";
6210                     delete $jobs->{$job};
6211                     next;
6212                 }
6213
6214                 die "$job: mirroring has been cancelled\n" if !defined($running_mirror_jobs->{$job});
6215
6216                 my $busy = $running_mirror_jobs->{$job}->{busy};
6217                 my $ready = $running_mirror_jobs->{$job}->{ready};
6218                 if (my $total = $running_mirror_jobs->{$job}->{len}) {
6219                     my $transferred = $running_mirror_jobs->{$job}->{offset} || 0;
6220                     my $remaining = $total - $transferred;
6221                     my $percent = sprintf "%.2f", ($transferred * 100 / $total);
6222
6223                     print "$job: transferred: $transferred bytes remaining: $remaining bytes total: $total bytes progression: $percent % busy: $busy ready: $ready \n";
6224                 }
6225
6226                 $readycounter++ if $running_mirror_jobs->{$job}->{ready};
6227             }
6228
6229             last if scalar(keys %$jobs) == 0;
6230
6231             if ($readycounter == scalar(keys %$jobs)) {
6232                 print "all mirroring jobs are ready \n";
6233                 last if $skipcomplete; #do the complete later
6234
6235                 if ($vmiddst && $vmiddst != $vmid) {
6236                     my $agent_running = $qga && qga_check_running($vmid);
6237                     if ($agent_running) {
6238                         print "freeze filesystem\n";
6239                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-freeze"); };
6240                     } else {
6241                         print "suspend vm\n";
6242                         eval { PVE::QemuServer::vm_suspend($vmid, 1); };
6243                     }
6244
6245                     # if we clone a disk for a new target vm, we don't switch the disk
6246                     PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs);
6247
6248                     if ($agent_running) {
6249                         print "unfreeze filesystem\n";
6250                         eval { PVE::QemuServer::vm_mon_cmd($vmid, "guest-fsfreeze-thaw"); };
6251                     } else {
6252                         print "resume vm\n";
6253                         eval {  PVE::QemuServer::vm_resume($vmid, 1, 1); };
6254                     }
6255
6256                     last;
6257                 } else {
6258
6259                     foreach my $job (keys %$jobs) {
6260                         # try to switch the disk if source and destination are on the same guest
6261                         print "$job: Completing block job...\n";
6262
6263                         eval { vm_mon_cmd($vmid, "block-job-complete", device => $job) };
6264                         if ($@ =~ m/cannot be completed/) {
6265                             print "$job: Block job cannot be completed, try again.\n";
6266                             $err_complete++;
6267                         }else {
6268                             print "$job: Completed successfully.\n";
6269                             $jobs->{$job}->{complete} = 1;
6270                         }
6271                     }
6272                 }
6273             }
6274             sleep 1;
6275         }
6276     };
6277     my $err = $@;
6278
6279     if ($err) {
6280         eval { PVE::QemuServer::qemu_blockjobs_cancel($vmid, $jobs) };
6281         die "mirroring error: $err";
6282     }
6283
6284 }
6285
6286 sub qemu_blockjobs_cancel {
6287     my ($vmid, $jobs) = @_;
6288
6289     foreach my $job (keys %$jobs) {
6290         print "$job: Cancelling block job\n";
6291         eval { vm_mon_cmd($vmid, "block-job-cancel", device => $job); };
6292         $jobs->{$job}->{cancel} = 1;
6293     }
6294
6295     while (1) {
6296         my $stats = vm_mon_cmd($vmid, "query-block-jobs");
6297
6298         my $running_jobs = {};
6299         foreach my $stat (@$stats) {
6300             $running_jobs->{$stat->{device}} = $stat;
6301         }
6302
6303         foreach my $job (keys %$jobs) {
6304
6305             if (defined($jobs->{$job}->{cancel}) && !defined($running_jobs->{$job})) {
6306                 print "$job: Done.\n";
6307                 delete $jobs->{$job};
6308             }
6309         }
6310
6311         last if scalar(keys %$jobs) == 0;
6312
6313         sleep 1;
6314     }
6315 }
6316
6317 sub clone_disk {
6318     my ($storecfg, $vmid, $running, $drivename, $drive, $snapname,
6319         $newvmid, $storage, $format, $full, $newvollist, $jobs, $skipcomplete, $qga) = @_;
6320
6321     my $newvolid;
6322
6323     if (!$full) {
6324         print "create linked clone of drive $drivename ($drive->{file})\n";
6325         $newvolid = PVE::Storage::vdisk_clone($storecfg,  $drive->{file}, $newvmid, $snapname);
6326         push @$newvollist, $newvolid;
6327     } else {
6328
6329         my ($storeid, $volname) = PVE::Storage::parse_volume_id($drive->{file});
6330         $storeid = $storage if $storage;
6331
6332         my $dst_format = resolve_dst_disk_format($storecfg, $storeid, $volname, $format);
6333         my ($size) = PVE::Storage::volume_size_info($storecfg, $drive->{file}, 3);
6334
6335         print "create full clone of drive $drivename ($drive->{file})\n";
6336         my $name = undef;
6337         if (drive_is_cloudinit($drive)) {
6338             $name = "vm-$newvmid-cloudinit";
6339             # cloudinit only supports raw and qcow2 atm:
6340             if ($dst_format eq 'qcow2') {
6341                 $name .= '.qcow2';
6342             } elsif ($dst_format ne 'raw') {
6343                 die "clone: unhandled format for cloudinit image\n";
6344             }
6345         }
6346         $newvolid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $newvmid, $dst_format, $name, ($size/1024));
6347         push @$newvollist, $newvolid;
6348
6349         PVE::Storage::activate_volumes($storecfg, [$newvolid]);
6350
6351         my $sparseinit = PVE::Storage::volume_has_feature($storecfg, 'sparseinit', $newvolid);
6352         if (!$running || $snapname) {
6353             qemu_img_convert($drive->{file}, $newvolid, $size, $snapname, $sparseinit);
6354         } else {
6355
6356             my $kvmver = get_running_qemu_version ($vmid);
6357             if (!qemu_machine_feature_enabled (undef, $kvmver, 2, 7)) {
6358                 die "drive-mirror with iothread requires qemu version 2.7 or higher\n"
6359                     if $drive->{iothread};
6360             }
6361
6362             qemu_drive_mirror($vmid, $drivename, $newvolid, $newvmid, $sparseinit, $jobs, $skipcomplete, $qga);
6363         }
6364     }
6365
6366     my ($size) = PVE::Storage::volume_size_info($storecfg, $newvolid, 3);
6367
6368     my $disk = $drive;
6369     $disk->{format} = undef;
6370     $disk->{file} = $newvolid;
6371     $disk->{size} = $size;
6372
6373     return $disk;
6374 }
6375
6376 # this only works if VM is running
6377 sub get_current_qemu_machine {
6378     my ($vmid) = @_;
6379
6380     my $cmd = { execute => 'query-machines', arguments => {} };
6381     my $res = vm_qmp_command($vmid, $cmd);
6382
6383     my ($current, $default);
6384     foreach my $e (@$res) {
6385         $default = $e->{name} if $e->{'is-default'};
6386         $current = $e->{name} if $e->{'is-current'};
6387     }
6388
6389     # fallback to the default machine if current is not supported by qemu
6390     return $current || $default || 'pc';
6391 }
6392
6393 sub get_running_qemu_version {
6394     my ($vmid) = @_;
6395     my $cmd = { execute => 'query-version', arguments => {} };
6396     my $res = vm_qmp_command($vmid, $cmd);
6397     return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
6398 }
6399
6400 sub qemu_machine_feature_enabled {
6401     my ($machine, $kvmver, $version_major, $version_minor) = @_;
6402
6403     my $current_major;
6404     my $current_minor;
6405
6406     if ($machine && $machine =~ m/^(pc(-i440fx|-q35)?-(\d+)\.(\d+))/) {
6407
6408         $current_major = $3;
6409         $current_minor = $4;
6410
6411     } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
6412
6413         $current_major = $1;
6414         $current_minor = $2;
6415     }
6416
6417     return 1 if $current_major >= $version_major && $current_minor >= $version_minor;
6418
6419
6420 }
6421
6422 sub qemu_machine_pxe {
6423     my ($vmid, $conf, $machine) = @_;
6424
6425     $machine =  PVE::QemuServer::get_current_qemu_machine($vmid) if !$machine;
6426
6427     foreach my $opt (keys %$conf) {
6428         next if $opt !~ m/^net(\d+)$/;
6429         my $net = PVE::QemuServer::parse_net($conf->{$opt});
6430         next if !$net;
6431         my $romfile = PVE::QemuServer::vm_mon_cmd_nocheck($vmid, 'qom-get', path => $opt, property => 'romfile');
6432         return $machine.".pxe" if $romfile =~ m/pxe/;
6433         last;
6434     }
6435
6436     return $machine;
6437 }
6438
6439 sub qemu_use_old_bios_files {
6440     my ($machine_type) = @_;
6441
6442     return if !$machine_type;
6443
6444     my $use_old_bios_files = undef;
6445
6446     if ($machine_type =~ m/^(\S+)\.pxe$/) {
6447         $machine_type = $1;
6448         $use_old_bios_files = 1;
6449     } else {
6450         my $kvmver = kvm_user_version();
6451         # Note: kvm version < 2.4 use non-efi pxe files, and have problems when we
6452         # load new efi bios files on migration. So this hack is required to allow
6453         # live migration from qemu-2.2 to qemu-2.4, which is sometimes used when
6454         # updrading from proxmox-ve-3.X to proxmox-ve 4.0
6455         $use_old_bios_files = !qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 4);
6456     }
6457
6458     return ($use_old_bios_files, $machine_type);
6459 }
6460
6461 sub create_efidisk {
6462     my ($storecfg, $storeid, $vmid, $fmt) = @_;
6463
6464     die "EFI vars default image not found\n" if ! -f $OVMF_VARS;
6465
6466     my $vars_size = PVE::Tools::convert_size(-s $OVMF_VARS, 'b' => 'kb');
6467     my $volid = PVE::Storage::vdisk_alloc($storecfg, $storeid, $vmid, $fmt, undef, $vars_size);
6468     PVE::Storage::activate_volumes($storecfg, [$volid]);
6469
6470     my $path = PVE::Storage::path($storecfg, $volid);
6471     eval {
6472         run_command(['/usr/bin/qemu-img', 'convert', '-n', '-f', 'raw', '-O', $fmt, $OVMF_VARS, $path]);
6473     };
6474     die "Copying EFI vars image failed: $@" if $@;
6475
6476     return ($volid, $vars_size);
6477 }
6478
6479 sub lspci {
6480
6481     my $devices = {};
6482
6483     dir_glob_foreach("$pcisysfs/devices", '[a-f0-9]{4}:([a-f0-9]{2}:[a-f0-9]{2})\.([0-9])', sub {
6484             my (undef, $id, $function) = @_;
6485             my $res = { id => $id, function => $function};
6486             push @{$devices->{$id}}, $res;
6487     });
6488
6489     # Entries should be sorted by functions.
6490     foreach my $id (keys %$devices) {
6491         my $dev = $devices->{$id};
6492         $devices->{$id} = [ sort { $a->{function} <=> $b->{function} } @$dev ];
6493     }
6494
6495     return $devices;
6496 }
6497
6498 sub vm_iothreads_list {
6499     my ($vmid) = @_;
6500
6501     my $res = vm_mon_cmd($vmid, 'query-iothreads');
6502
6503     my $iothreads = {};
6504     foreach my $iothread (@$res) {
6505         $iothreads->{ $iothread->{id} } = $iothread->{"thread-id"};
6506     }
6507
6508     return $iothreads;
6509 }
6510
6511 sub scsihw_infos {
6512     my ($conf, $drive) = @_;
6513
6514     my $maxdev = 0;
6515
6516     if (!$conf->{scsihw} || ($conf->{scsihw} =~ m/^lsi/)) {
6517         $maxdev = 7;
6518     } elsif ($conf->{scsihw} && ($conf->{scsihw} eq 'virtio-scsi-single')) {
6519         $maxdev = 1;
6520     } else {
6521         $maxdev = 256;
6522     }
6523
6524     my $controller = int($drive->{index} / $maxdev);
6525     my $controller_prefix = ($conf->{scsihw} && $conf->{scsihw} eq 'virtio-scsi-single') ? "virtioscsi" : "scsihw";
6526
6527     return ($maxdev, $controller, $controller_prefix);
6528 }
6529
6530 sub add_hyperv_enlightenments {
6531     my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios, $gpu_passthrough) = @_;
6532
6533     return if $winversion < 6;
6534     return if $bios && $bios eq 'ovmf' && $winversion < 8;
6535
6536     push @$cpuFlags , 'hv_vendor_id=proxmox' if $gpu_passthrough;
6537
6538     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
6539         push @$cpuFlags , 'hv_spinlocks=0x1fff';
6540         push @$cpuFlags , 'hv_vapic';
6541         push @$cpuFlags , 'hv_time';
6542     } else {
6543         push @$cpuFlags , 'hv_spinlocks=0xffff';
6544     }
6545
6546     if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
6547         push @$cpuFlags , 'hv_reset';
6548         push @$cpuFlags , 'hv_vpindex';
6549         push @$cpuFlags , 'hv_runtime';
6550     }
6551
6552     if ($winversion >= 7) {
6553         push @$cpuFlags , 'hv_relaxed';
6554     }
6555 }
6556
6557 sub windows_version {
6558     my ($ostype) = @_;
6559
6560     return 0 if !$ostype;
6561
6562     my $winversion = 0;
6563
6564     if($ostype eq 'wxp' || $ostype eq 'w2k3' || $ostype eq 'w2k') {
6565         $winversion = 5;
6566     } elsif($ostype eq 'w2k8' || $ostype eq 'wvista') {
6567         $winversion = 6;
6568     } elsif ($ostype =~ m/^win(\d+)$/) {
6569         $winversion = $1;
6570     }
6571
6572     return $winversion;
6573 }
6574
6575 sub resolve_dst_disk_format {
6576         my ($storecfg, $storeid, $src_volname, $format) = @_;
6577         my ($defFormat, $validFormats) = PVE::Storage::storage_default_format($storecfg, $storeid);
6578
6579         if (!$format) {
6580             # if no target format is specified, use the source disk format as hint
6581             if ($src_volname) {
6582                 my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
6583                 $format = qemu_img_format($scfg, $src_volname);
6584             } else {
6585                 return $defFormat;
6586             }
6587         }
6588
6589         # test if requested format is supported - else use default
6590         my $supported = grep { $_ eq $format } @$validFormats;
6591         $format = $defFormat if !$supported;
6592         return $format;
6593 }
6594
6595 sub resolve_first_disk {
6596     my $conf = shift;
6597     my @disks = PVE::QemuServer::valid_drive_names();
6598     my $firstdisk;
6599     foreach my $ds (reverse @disks) {
6600         next if !$conf->{$ds};
6601         my $disk = PVE::QemuServer::parse_drive($ds, $conf->{$ds});
6602         next if PVE::QemuServer::drive_is_cdrom($disk);
6603         $firstdisk = $ds;
6604     }
6605     return $firstdisk;
6606 }
6607
6608 sub generate_smbios1_uuid {
6609     my ($uuid, $uuid_str);
6610     UUID::generate($uuid);
6611     UUID::unparse($uuid, $uuid_str);
6612     return "uuid=$uuid_str";
6613 }
6614
6615 # bash completion helper
6616
6617 sub complete_backup_archives {
6618     my ($cmdname, $pname, $cvalue) = @_;
6619
6620     my $cfg = PVE::Storage::config();
6621
6622     my $storeid;
6623
6624     if ($cvalue =~ m/^([^:]+):/) {
6625         $storeid = $1;
6626     }
6627
6628     my $data = PVE::Storage::template_list($cfg, $storeid, 'backup');
6629
6630     my $res = [];
6631     foreach my $id (keys %$data) {
6632         foreach my $item (@{$data->{$id}}) {
6633             next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
6634             push @$res, $item->{volid} if defined($item->{volid});
6635         }
6636     }
6637
6638     return $res;
6639 }
6640
6641 my $complete_vmid_full = sub {
6642     my ($running) = @_;
6643
6644     my $idlist = vmstatus();
6645
6646     my $res = [];
6647
6648     foreach my $id (keys %$idlist) {
6649         my $d = $idlist->{$id};
6650         if (defined($running)) {
6651             next if $d->{template};
6652             next if $running && $d->{status} ne 'running';
6653             next if !$running && $d->{status} eq 'running';
6654         }
6655         push @$res, $id;
6656
6657     }
6658     return $res;
6659 };
6660
6661 sub complete_vmid {
6662     return &$complete_vmid_full();
6663 }
6664
6665 sub complete_vmid_stopped {
6666     return &$complete_vmid_full(0);
6667 }
6668
6669 sub complete_vmid_running {
6670     return &$complete_vmid_full(1);
6671 }
6672
6673 sub complete_storage {
6674
6675     my $cfg = PVE::Storage::config();
6676     my $ids = $cfg->{ids};
6677
6678     my $res = [];
6679     foreach my $sid (keys %$ids) {
6680         next if !PVE::Storage::storage_check_enabled($cfg, $sid, undef, 1);
6681         next if !$ids->{$sid}->{content}->{images};
6682         push @$res, $sid;
6683     }
6684
6685     return $res;
6686 }
6687
6688 1;